merge main into Lens isolation and preserve signals endpoints

.github/codeql/codeql-config.yml

@@ -1,8 +1,5 @@
 name: "LiteLLM CodeQL config"
 
-queries:
-  - uses: security-and-quality
-
 # Known OOM queries on large Python codebases:
 # CodeQL builds a full data flow graph in memory. These two queries trace
 # sensitive data through every log call / regex pattern, causing combinatorial
@@ -14,17 +11,6 @@ query-filters:
       id: py/clear-text-logging-sensitive-data  # CWE-312
   - exclude:
       id: py/polynomial-redos                   # CWE-730
-  # Import resolution confuses stdlib types with management_endpoints/types.py.
-  # The generic cycle query also reports intentional deferred imports.
-  - exclude:
-      id: py/cyclic-import
-  - exclude:
-      id: py/unsafe-cyclic-import
-  # Known false positives on live settings and Protocol placeholders.
-  - exclude:
-      id: py/unused-global-variable
-  - exclude:
-      id: py/ineffectual-statement
 
 paths-ignore:
   - tests

.github/workflows/codeql.yml

@@ -1,8 +1,6 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main]
   schedule:
@@ -43,14 +41,15 @@ jobs:
           persist-credentials: false
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
+        uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           languages: ${{ matrix.language }}
           build-mode: ${{ matrix.build-mode }}
           config-file: ./.github/codeql/codeql-config.yml
+          queries: ${{ github.event_name == 'pull_request' && '+security-extended' || '' }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
+        uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           category: "/language:${{ matrix.language }}"
           output: sarif-results
@@ -83,7 +82,7 @@ jobs:
           output: sarif-results/python.sarif
 
       - name: Upload SARIF
-        uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
+        uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           sarif_file: sarif-results
           category: "/language:${{ matrix.language }}"

litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000200_lens_trace_signals/migration.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_LensSignalConfig" (
+    "id" TEXT NOT NULL,
+    "data" JSONB NOT NULL,
+    CONSTRAINT "LiteLLM_LensSignalConfig_pkey" PRIMARY KEY ("id")
+);
+
+CREATE TABLE IF NOT EXISTS "LiteLLM_LensTraceSignal" (
+    "trace_id" TEXT NOT NULL,
+    "trace_ref" TEXT NOT NULL DEFAULT '',
+    "config_key" TEXT NOT NULL,
+    "span_count" INTEGER NOT NULL,
+    "claimed_until" TIMESTAMP(3),
+    "classified_at" TIMESTAMP(3),
+    "data" JSONB NOT NULL,
+    CONSTRAINT "LiteLLM_LensTraceSignal_pkey" PRIMARY KEY ("trace_id", "trace_ref")
+);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1982,3 +1982,20 @@ model LiteLLM_LensDataset {
 
   @@id([id, revision])
 }
+
+model LiteLLM_LensSignalConfig {
+  id   String @id
+  data Json
+}
+
+model LiteLLM_LensTraceSignal {
+  trace_id      String
+  trace_ref     String @default("")
+  config_key    String
+  span_count    Int
+  claimed_until DateTime?
+  classified_at DateTime?
+  data          Json
+
+  @@id([trace_id, trace_ref])
+}

litellm-rust/crates/traces-clickhouse/query/lens_content.sql

@@ -17,6 +17,7 @@ SELECT * FROM (
     FROM otel_traces WHERE {source:String}='traces'
       AND ({all_teams:UInt8}=1 OR TeamId={team:String})
       AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY
       AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String})
       AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId > {cursor:String}
     ORDER BY SpanId LIMIT 1 BY SpanId LIMIT 40
@@ -35,5 +36,6 @@ SELECT * FROM (
     FROM spend_logs FINAL WHERE {source:String}='requests'
       AND ({all_teams:UInt8}=1 OR team_id={team:String})
       AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY
       AND request_id={id:String} AND team_id={record_team:String} LIMIT 1
 )

litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql

@@ -2,6 +2,7 @@ SELECT sum(matches) AS count FROM (
     SELECT count() AS matches FROM otel_traces WHERE {source:String}='traces'
       AND ({all_teams:UInt8}=1 OR TeamId={team:String})
       AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY
       AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String})
       AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId={span:String}
       AND position(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage),{quote:String})>0
@@ -9,6 +10,7 @@ SELECT sum(matches) AS count FROM (
     SELECT count() AS matches FROM spend_logs FINAL WHERE {source:String}='requests'
       AND ({all_teams:UInt8}=1 OR team_id={team:String})
       AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY
       AND request_id={id:String} AND team_id={record_team:String} AND request_id={span:String}
       AND position(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str),{quote:String})>0
 )

litellm-rust/crates/traces-clickhouse/src/query/lens.rs

@@ -209,6 +209,7 @@ pub struct LensContentParams {
     pub source: ContentSource,
     pub id: String,
     pub record_team: String,
+    pub start_time: String,
     pub trace_ref: String,
     pub cursor: String,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -252,6 +253,7 @@ pub struct LensEvidenceParams {
     pub source: ContentSource,
     pub id: String,
     pub record_team: String,
+    pub start_time: String,
     pub trace_ref: String,
     pub span: String,
     pub quote: String,

litellm-rust/crates/traces-clickhouse/src/query/number.rs

@@ -99,7 +99,7 @@ mod tests {
     fn content_rejects_unsupported_sources(#[case] source: &str, #[case] valid: bool) {
         let parameters = serde_json::json!({
             "all_teams": 0, "team": "team", "key_hash": "", "source": source, "id": "id",
-            "record_team": "team", "trace_ref": "", "cursor": "", "offset": 0
+            "record_team": "team", "start_time": "", "trace_ref": "", "cursor": "", "offset": 0
         });
         assert_eq!(
             serde_json::from_value::<crate::query::lens::LensContentParams>(parameters).is_ok(),

litellm-rust/crates/traces-clickhouse/tests/load.rs

@@ -25,7 +25,8 @@ async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestR
         "INSERT INTO {DATABASE}.otel_traces \
          (Timestamp, TraceId, SpanId, ParentSpanId, SpanName, ServiceName, ObservationType, TeamId, ApiKeyHash, Duration, SpanAttributes) \
          SELECT now64(9) - toIntervalHour(intDiv(number, {SPANS_PER_DAY}) * 24 + 12 + {first_day} * 24), \
-         concat('load-', toString(number + {first_row})), concat('span-', toString(number + {first_row})), \
+         if({first_day} = 0, concat('load-', toString(number + {first_row})), 'load-0'), \
+         concat('span-', toString(number + {first_row})), \
          '', 'span', 'service', 'agent', 'load-team', '', 0, \
          if({first_day}=0 AND number < {SPANS_PER_DAY}, map('payload', repeat('x', 3000)), map()) \
          FROM numbers({count})"
@@ -41,6 +42,62 @@ async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestR
     Ok(())
 }
 
+async fn trace_start_time(fixture: &SeededDatabase) -> TestResult<String> {
+    let query = format!(
+        "SELECT toString(Timestamp, 'UTC') AS start_time FROM {DATABASE}.otel_traces \
+         WHERE TraceId = 'load-0' LIMIT 1 FORMAT JSON"
+    );
+    let response = fixture
+        .database
+        .client
+        .post(&fixture.database.url)
+        .body(query)
+        .send()
+        .await?
+        .error_for_status()?
+        .text()
+        .await?;
+    let result: Value = serde_json::from_str(&response)?;
+    result["data"][0]["start_time"]
+        .as_str()
+        .map(str::to_owned)
+        .ok_or_else(|| "trace start time missing".into())
+}
+
+fn content_parameters(start_time: &str) -> BTreeMap<String, Parameter> {
+    BTreeMap::from([
+        ("source".into(), Parameter::Text("traces".into())),
+        ("all_teams".into(), Parameter::Integer(0)),
+        ("team".into(), Parameter::Text("load-team".into())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        ("id".into(), Parameter::Text("load-0".into())),
+        ("record_team".into(), Parameter::Text("load-team".into())),
+        ("start_time".into(), Parameter::Text(start_time.into())),
+        ("trace_ref".into(), Parameter::Text(String::new())),
+        ("cursor".into(), Parameter::Text(String::new())),
+        ("offset".into(), Parameter::Integer(1)),
+    ])
+}
+
+async fn content(fixture: &SeededDatabase, start_time: &str, query_id: &str) -> TestResult {
+    let connection = Connection::configured(
+        &format!("{}?query_id={query_id}", fixture.database.url),
+        DATABASE,
+        "default",
+        "",
+    )?;
+    let response = execute_named_read(
+        &fixture.database.client,
+        &connection,
+        ReadQuery::Content,
+        &content_parameters(start_time),
+    )
+    .await?;
+    let result: Value = serde_json::from_str(&response)?;
+    assert!(!result["data"].as_array().ok_or("content rows")?.is_empty());
+    Ok(())
+}
+
 fn sample_parameters(start: u64, end: u64) -> BTreeMap<String, Parameter> {
     BTreeMap::from([
         ("source".into(), Parameter::Text("traces".into())),
@@ -144,3 +201,27 @@ async fn lens_sample_reads_scale_with_window_not_retention(
     );
     Ok(())
 }
+
+#[rstest]
+#[tokio::test]
+async fn lens_content_reads_scale_with_trace_not_retention(
+    #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+) -> TestResult {
+    let fixture = migrated_database?;
+    seed_days(&fixture, 0, 8).await?;
+    let start_time = trace_start_time(&fixture).await?;
+    let before_id = format!("lens_content_before_{}", std::process::id());
+    content(&fixture, &start_time, &before_id).await?;
+    let before = query_read_rows(&fixture, &before_id).await?;
+
+    seed_days(&fixture, 8, 24).await?;
+    let after_id = format!("lens_content_after_{}", std::process::id());
+    content(&fixture, &start_time, &after_id).await?;
+    let after = query_read_rows(&fixture, &after_id).await?;
+    println!("lens_content read_rows: before={before}, after={after}");
+    assert!(
+        after * 100 <= before * 105,
+        "read_rows grew from {before} to {after}"
+    );
+    Ok(())
+}

litellm-rust/crates/traces-clickhouse/tests/migrations.rs

@@ -1150,6 +1150,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate(
         ("source".into(), Parameter::Text("traces".into())),
         ("id".into(), Parameter::Text("shared".into())),
         ("record_team".into(), Parameter::Text("team".into())),
+        ("start_time".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(first_ref.into())),
         ("cursor".into(), Parameter::Text(String::new())),
         ("offset".into(), Parameter::Integer(1)),
@@ -1177,6 +1178,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate(
         ("source".into(), Parameter::Text("traces".into())),
         ("id".into(), Parameter::Text("shared".into())),
         ("record_team".into(), Parameter::Text("team".into())),
+        ("start_time".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(first_ref.into())),
         ("span".into(), Parameter::Text("root".into())),
         ("quote".into(), Parameter::Text(opposite.into())),
@@ -1339,7 +1341,7 @@ async fn lens_selection_pages_without_losing_or_repeating_runs(
 #[case::traces("traces", 9)]
 #[case::requests("requests", 3)]
 #[tokio::test]
-async fn lens_content_keeps_original_span_and_request_timestamps(
+async fn lens_content_keeps_original_timestamps_with_start_time_slack(
     #[future(awt)] database: TestResult<ClickHouseDatabase>,
     #[case] source: &str,
     #[case] precision: usize,
@@ -1379,6 +1381,30 @@ async fn lens_content_keeps_original_span_and_request_timestamps(
     )
     .await?;
     let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let start_time_body = execute_read(
+        &database.client,
+        &connection,
+        "SELECT toString(fromUnixTimestamp64Nano({timestamp:Int64})) AS start_time FORMAT JSON",
+        &BTreeMap::from([(
+            "timestamp".into(),
+            Parameter::Integer(root_start + 86_400_000_000_000),
+        )]),
+    )
+    .await?;
+    let start_time: serde_json::Value = serde_json::from_str(&start_time_body)?;
+    let start_time = start_time["data"][0]["start_time"]
+        .as_str()
+        .ok_or("start time missing")?
+        .to_owned();
+    let parsed_time_body = execute_read(
+        &database.client,
+        &connection,
+        "SELECT toString(parseDateTime64BestEffortOrZero({start_time:String}, 9)) AS start_time FORMAT JSON",
+        &BTreeMap::from([("start_time".into(), Parameter::Text(start_time.clone()))]),
+    )
+    .await?;
+    let parsed_time: serde_json::Value = serde_json::from_str(&parsed_time_body)?;
+    assert_eq!(parsed_time["data"][0]["start_time"], start_time);
     let parameters = BTreeMap::from([
         ("source".into(), Parameter::Text(source.into())),
         ("all_teams".into(), Parameter::Integer(0)),
@@ -1386,6 +1412,7 @@ async fn lens_content_keeps_original_span_and_request_timestamps(
         ("record_team".into(), Parameter::Text("team".into())),
         ("key_hash".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(String::new())),
+        ("start_time".into(), Parameter::Text(start_time)),
         ("id".into(), Parameter::Text("run".into())),
         ("cursor".into(), Parameter::Text(String::new())),
         ("offset".into(), Parameter::Integer(1)),
@@ -1453,6 +1480,7 @@ async fn lens_content_keeps_output_visible_after_long_input(
         ("record_team".into(), Parameter::Text("team".into())),
         ("key_hash".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(String::new())),
+        ("start_time".into(), Parameter::Text(String::new())),
         ("id".into(), Parameter::Text("request".into())),
         ("cursor".into(), Parameter::Text(String::new())),
         ("offset".into(), Parameter::Integer(1)),

litellm/integrations/otel/model/spans.py

@@ -354,6 +354,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_LensReview",
         "LiteLLM_LensIngestionKey",
         "LiteLLM_LensWorker",
+        "LiteLLM_LensSignalConfig",
+        "LiteLLM_LensTraceSignal",
     )
 )
 PRISMA_RELATIONS: Final[frozenset[str]] = _PRISMA_MODELS | _PRISMA_VIEWS

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -5188,7 +5188,7 @@ def function_call_prompt(
     messages: list[dict[str, object]],
     functions: list[object],
 ) -> list[dict[str, object]]:
-    function_prompt = """Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}} The following functions are available to you:"""
+    function_prompt = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
     for function in functions:
         function_prompt += f"""\n{function}\n"""
 

litellm/llms/base_llm/chat/transformation.py

@@ -65,6 +65,26 @@ class BaseLLMException(Exception):
         super().__init__(self.message)  # Call the base class constructor with the parameters it needs
 
 
+_NO_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
+
+
+def with_attribution_headers(
+    attribution_headers: Mapping[str, str],
+    headers: dict[str, str] | None,  # mutable-ok: returned as-is when there is nothing to add
+) -> dict[str, str] | None:  # mutable-ok: becomes the request's outbound headers
+    """
+    `headers` plus any attribution header the caller didn't already set (names
+    compared case-insensitively). Builds a new dict; `headers` is never mutated.
+    """
+    if not attribution_headers:
+        return headers
+    caller_names: Final = {name.lower() for name in headers or {}}
+    return {
+        **{name: value for name, value in attribution_headers.items() if name.lower() not in caller_names},
+        **(headers or {}),
+    }
+
+
 class BaseConfig(ABC):
     def __init__(self):
         pass
@@ -89,6 +109,15 @@ class BaseConfig(ABC):
             and not callable(v)  # Filter out any callable objects including mocks
         }
 
+    def get_attribution_headers(self) -> Mapping[str, str]:
+        """
+        Headers that tell the provider a request came through LiteLLM.
+
+        Sent by default on every request; a caller header with the same name
+        (any casing) wins. Override in a provider config to opt in.
+        """
+        return _NO_ATTRIBUTION_HEADERS
+
     def get_json_schema_from_pydantic_object(self, response_format: type[BaseModel] | dict | None) -> dict | None:
         return type_to_response_format_param(response_format=response_format)
 

litellm/llms/novita/chat/transformation.py

@@ -6,11 +6,20 @@ Calls done in OpenAI/openai.py as Novita AI is openai-compatible.
 Docs: https://novita.ai/docs/guides/llm-api
 """
 
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
 from ....types.llms.openai import AllMessageValues
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
+_NOVITA_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"X-Novita-Source": "litellm"})
+
 
 class NovitaConfig(OpenAIGPTConfig):
+    def get_attribution_headers(self) -> Mapping[str, str]:
+        return _NOVITA_ATTRIBUTION_HEADERS
+
     def validate_environment(
         self,
         headers: dict,
@@ -27,5 +36,6 @@ class NovitaConfig(OpenAIGPTConfig):
             )
         headers["Authorization"] = f"Bearer {api_key}"
         headers["Content-Type"] = "application/json"
-        headers["X-Novita-Source"] = "litellm"
+        if not any(name.lower() == "x-novita-source" for name in headers):
+            headers["X-Novita-Source"] = "litellm"
         return headers

litellm/llms/perplexity/chat/transformation.py

@@ -2,6 +2,8 @@
 Translate from OpenAI's `/v1/chat/completions` to Perplexity's `/v1/chat/completions`
 """
 
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
@@ -17,12 +19,17 @@ from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
+_PERPLEXITY_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"X-Pplx-Integration": "litellm"})
+
 
 class PerplexityChatConfig(OpenAIGPTConfig):
     @property
     def custom_llm_provider(self) -> str | None:
         return "perplexity"
 
+    def get_attribution_headers(self) -> Mapping[str, str]:
+        return _PERPLEXITY_ATTRIBUTION_HEADERS
+
     def _get_openai_compatible_provider_info(
         self, api_base: str | None, api_key: str | None
     ) -> tuple[str | None, str | None]:

litellm/main.py

@@ -118,6 +118,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
 from litellm.llms.base_llm.base_model_iterator import (
     convert_model_response_to_streaming,
 )
+from litellm.llms.base_llm.chat.transformation import with_attribution_headers
 from litellm.llms.bedrock.common_utils import (
     BedrockModelInfo,
     bedrock_route_for_request,
@@ -2634,6 +2635,11 @@ def _complete_custom_openai(
     )
 
     headers = headers or litellm.headers
+    outbound_headers: Final = (
+        headers
+        if provider_config is None
+        else with_attribution_headers(provider_config.get_attribution_headers(), headers)
+    )
 
     # Add GitHub Copilot headers (same as /responses endpoint does)
     if custom_llm_provider == "github_copilot":
@@ -2685,7 +2691,7 @@ def _complete_custom_openai(
                 acompletion=acompletion,
                 stream=stream,
                 api_key=api_key,
-                headers=headers,
+                headers=outbound_headers,
                 client=client,
                 provider_config=provider_config,
             )
@@ -2693,7 +2699,7 @@ def _complete_custom_openai(
             response = openai_chat_completions.completion(
                 model=model,
                 messages=messages,
-                headers=headers,
+                headers=outbound_headers,
                 model_response=model_response,
                 print_verbose=print_verbose,
                 api_key=api_key,
@@ -2716,7 +2722,7 @@ def _complete_custom_openai(
             input=messages,
             api_key=api_key,
             original_response=str(e),
-            additional_args={"headers": headers},
+            additional_args={"headers": outbound_headers},
         )
         raise e
 
@@ -2726,7 +2732,7 @@ def _complete_custom_openai(
             input=messages,
             api_key=api_key,
             original_response=response,
-            additional_args={"headers": headers},
+            additional_args={"headers": outbound_headers},
         )
 
     return response  # pyright: ignore[reportReturnType]  # provider SDK return type is broader than the dispatch contract

litellm/proxy/lens/endpoints.py

@@ -62,6 +62,8 @@ from litellm.proxy.lens.models import (
 from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_image
 from litellm.proxy.lens.repository import DueLens, LensRepository, WriterDatabase
 from litellm.proxy.lens.reviews import criteria_key
+from litellm.proxy.lens.signal_repository import SignalRepository
+from litellm.proxy.lens.signals import SignalConfig, TraceSignals, trace_signals
 from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
 from litellm.proxy.lens.state import (
     can_access,
@@ -79,6 +81,7 @@ from litellm.proxy.lens.state import (
     summarized,
 )
 from litellm.proxy.tracing_runtime import provide_storage
+from litellm.router import Router
 from litellm.tracing.remote import LensConnection, bounded_response
 from litellm.types.llms.base import LiteLLMBaseModel
 
@@ -112,6 +115,14 @@ def repository() -> LensRepository:
     return LensRepository(WriterDatabase(writer_wrapper(prisma_client.db)))
 
 
+def signals_repository() -> SignalRepository:
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(503, "Lens needs a connected Postgres database")
+    return SignalRepository(WriterDatabase(writer_wrapper(prisma_client.db)))
+
+
 def source_reader(storage: Storage | None) -> SourceReader:
     if storage is None:
         raise HTTPException(
@@ -129,6 +140,20 @@ def user_scope(auth: UserAPIKeyAuth, write: bool = False) -> Scope:
     raise HTTPException(403, "Lens requires proxy administrator access")
 
 
+def validate_signal_model(config: SignalConfig, llm_router: Router | None) -> None:
+    if not config.model:
+        return
+    message: Final = "Choose a System 1 model (evaluation mode) configured on this proxy"
+    if llm_router is None:
+        raise HTTPException(400, message)
+    try:
+        model_group: Final = llm_router.get_model_group_info(model_group=config.model)
+    except Exception as error:
+        raise HTTPException(400, message) from error
+    if model_group is None or model_group.mode != "evaluation":
+        raise HTTPException(400, message)
+
+
 async def get_lens(lens_id: str, scope: Scope) -> Lens:
     lens: Final = await repository().get(lens_id)
     if lens is None or not can_access(scope, lens.scope):
@@ -367,6 +392,39 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]:
     return await source_reader(storage).agents(scope) if storage is not None else ()
 
 
+@router.get("/signals", response_model=SignalConfig)
+async def get_signals(auth: Auth) -> SignalConfig:
+    user_scope(auth)
+    return await signals_repository().get_config()
+
+
+@router.put("/signals", response_model=SignalConfig)
+async def put_signals(body: SignalConfig, auth: Auth) -> SignalConfig:
+    user_scope(auth, write=True)
+    from litellm.proxy.proxy_server import llm_router
+
+    validate_signal_model(body, llm_router)
+    await signals_repository().save_config(body)
+    return body
+
+
+@router.post("/traces/signals", response_model=tuple[TraceSignals, ...])
+async def trace_signal_statuses(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceSignals, ...]:
+    user_scope(auth)
+    repo: Final = signals_repository()
+    config: Final = await repo.get_config()
+    existing: Final = await repo.traces(body.traces)
+    rows: Final = MappingProxyType({(row.trace_id, row.trace_ref): row for row in existing})
+    return tuple(
+        trace_signals(
+            trace,
+            rows.get((trace.trace_id, trace.trace_ref)),
+            config,
+        )
+        for trace in body.traces
+    )
+
+
 @router.post("/traces/findings", response_model=tuple[TraceFindingCount, ...])
 async def trace_findings(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceFindingCount, ...]:
     user_scope(auth)

litellm/proxy/lens/signal_repository.py

@@ -0,0 +1,142 @@
+import json
+from datetime import datetime
+from typing import Final
+
+from pydantic import TypeAdapter
+
+from litellm.proxy.lens.models import Execution, TraceIdentity
+from litellm.proxy.lens.repository import Database, Row
+from litellm.proxy.lens.signals import (
+    SIGNAL_RECLASSIFY_AFTER,
+    SIGNAL_RETRY_FAILED_AFTER,
+    SignalAttempt,
+    SignalConfig,
+    StoredTraceSignal,
+)
+
+_ROWS: Final[TypeAdapter[tuple[Row, ...]]] = TypeAdapter(tuple[Row, ...])
+
+
+class SignalRepository:
+    def __init__(self, db: Database) -> None:
+        self.db: Final = db
+
+    async def get_config(self) -> SignalConfig:
+        rows: Final = _ROWS.validate_python(
+            await self.db.query_raw('SELECT data FROM "LiteLLM_LensSignalConfig" WHERE id=$1', "global")
+        )
+        return SignalConfig() if not rows else SignalConfig.model_validate(rows[0].data)
+
+    async def save_config(self, config: SignalConfig) -> None:
+        await self.db.execute_raw(
+            """INSERT INTO "LiteLLM_LensSignalConfig" (id, data)
+            VALUES ($1, $2::jsonb)
+            ON CONFLICT (id) DO UPDATE SET data=EXCLUDED.data""",
+            "global",
+            json.dumps(config.model_dump(mode="json")),
+        )
+
+    async def traces(self, identities: tuple[TraceIdentity, ...]) -> tuple[StoredTraceSignal, ...]:
+        if not identities:
+            return ()
+        payload: Final = json.dumps(
+            tuple({"trace_id": trace.trace_id, "trace_ref": trace.trace_ref} for trace in identities)
+        )
+        rows: Final = _ROWS.validate_python(
+            await self.db.query_raw(
+                """SELECT jsonb_build_object(
+                    'trace_id', trace_id,
+                    'trace_ref', trace_ref,
+                    'config_key', config_key,
+                    'span_count', span_count,
+                    'claimed_until', claimed_until,
+                    'classified_at', classified_at,
+                    'data', data
+                ) AS data
+                FROM "LiteLLM_LensTraceSignal"
+                WHERE (trace_id, trace_ref) IN (
+                    SELECT trace_id, trace_ref FROM jsonb_to_recordset($1::jsonb) AS requested(
+                        trace_id text, trace_ref text
+                    )
+                )""",
+                payload,
+            )
+        )
+        return tuple(StoredTraceSignal.model_validate(row.data) for row in rows)
+
+    async def claim(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        now: datetime,
+    ) -> bool:
+        data: Final = json.dumps({"status": "pending", "scores": {}, "model": config.model, "error": ""})
+        rows: Final = _ROWS.validate_python(
+            await self.db.query_raw(
+                """INSERT INTO "LiteLLM_LensTraceSignal" AS stored
+                    (trace_id, trace_ref, config_key, span_count, claimed_until, classified_at, data)
+                VALUES ($1, $2, $3, $4, $5::timestamp, NULL, $6::jsonb)
+                ON CONFLICT (trace_id, trace_ref) DO UPDATE SET
+                    config_key=EXCLUDED.config_key,
+                    span_count=EXCLUDED.span_count,
+                    claimed_until=EXCLUDED.claimed_until,
+                    classified_at=NULL,
+                    data=EXCLUDED.data
+                WHERE (stored.claimed_until IS NULL OR stored.claimed_until < $7::timestamp)
+                    AND (
+                        stored.config_key IS DISTINCT FROM EXCLUDED.config_key
+                        OR (
+                            stored.data->>'status'='pending'
+                            AND stored.claimed_until < $7::timestamp
+                        )
+                        OR (
+                            EXCLUDED.span_count > stored.span_count
+                            AND stored.classified_at < $8::timestamp
+                        )
+                        OR (
+                            stored.data->>'status'='failed'
+                            AND stored.classified_at < $9::timestamp
+                        )
+                    )
+                RETURNING jsonb_build_object('trace_id', trace_id) AS data""",
+                execution.trace_id,
+                execution.trace_ref,
+                config.key(),
+                execution.span_count,
+                claimed_until,
+                data,
+                now,
+                now - SIGNAL_RECLASSIFY_AFTER,
+                now - SIGNAL_RETRY_FAILED_AFTER,
+            )
+        )
+        return bool(rows)
+
+    async def store(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        classified_at: datetime,
+        attempt: SignalAttempt,
+    ) -> None:
+        payload: Final = json.dumps(
+            {
+                "status": attempt.status,
+                "scores": dict(attempt.scores),
+                "model": attempt.model,
+                "error": attempt.error,
+            }
+        )
+        await self.db.execute_raw(
+            """UPDATE "LiteLLM_LensTraceSignal"
+            SET classified_at=$1::timestamp, claimed_until=NULL, data=$2::jsonb
+            WHERE trace_id=$3 AND trace_ref=$4 AND config_key=$5 AND claimed_until=$6::timestamp""",
+            classified_at,
+            payload,
+            execution.trace_id,
+            execution.trace_ref,
+            config.key(),
+            claimed_until,
+        )

litellm/proxy/lens/signals.py

@@ -0,0 +1,586 @@
+import asyncio
+import hashlib
+import json
+from collections.abc import Callable, Mapping
+from datetime import datetime, timedelta, timezone
+from itertools import accumulate
+from types import MappingProxyType
+from typing import Annotated, Final, Literal, Protocol, TypeAlias
+
+from pydantic import ConfigDict, Field, JsonValue, ValidationError, field_validator, model_validator
+
+from litellm.integrations.clickhouse.context import lens_analysis
+from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
+from litellm.litellm_core_utils.secret_redaction import redact_internal_details
+from litellm.proxy.lens.models import ActivitySelection, Execution, Record, Scope, TraceIdentity
+from litellm.proxy.lens.sources import SourceReader, Storage
+
+SIGNAL_INTERVAL_SECONDS: Final = 60
+SIGNAL_PAGE_SIZE: Final = 100
+SIGNAL_MAX_PER_TICK: Final = 50
+SIGNAL_CONCURRENCY: Final = 8
+SIGNAL_CLAIM_LEASE: Final = timedelta(minutes=5)
+SIGNAL_RECLASSIFY_AFTER: Final = timedelta(minutes=5)
+SIGNAL_RETRY_FAILED_AFTER: Final = timedelta(minutes=30)
+SIGNAL_MAX_CONTENT_PAGES: Final = 3
+SIGNAL_PART_MAX_CHARS: Final = 2000
+SIGNAL_PART_HEAD_CHARS: Final = 800
+SIGNAL_PART_TAIL_CHARS: Final = 1200
+SIGNAL_TRANSCRIPT_MAX_CHARS: Final = 40000
+SIGNAL_TRANSCRIPT_HEAD_CHARS: Final = 15000
+SIGNAL_TRANSCRIPT_TAIL_CHARS: Final = 25000
+SIGNAL_MAX_SCAN_PAGES: Final = 10
+SIGNAL_TASK: Final = (
+    "An AI agent run recorded as a trace. Judge only what the user and the agent said and did in these steps."
+)
+
+
+class Signal(Record):
+    id: str = Field(pattern=r"^[a-z][a-z0-9_]{0,63}$")
+    name: str = Field(min_length=1, max_length=60)
+    question: str = Field(min_length=3, max_length=500)
+
+
+DEFAULT_SIGNALS: Final[tuple[Signal, ...]] = (
+    Signal(
+        id="user_frustration",
+        name="User frustration",
+        question=(
+            "Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example "
+            "complaints, irritated corrections, all caps, profanity, or giving up on the task?"
+        ),
+    ),
+    Signal(
+        id="missing_capability",
+        name="Missing capability",
+        question=(
+            "Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it "
+            "lacks a tool, permission, integration or data source, or fails because the capability does not exist?"
+        ),
+    ),
+    Signal(
+        id="repeated_request",
+        name="Repeated request",
+        question=(
+            "Does the user ask for the same thing more than once in this run, usually because the agent did not "
+            "deliver it the first time?"
+        ),
+    ),
+)
+
+
+class SignalConfig(Record):
+    model: str = ""
+    threshold: float = Field(default=0.5, ge=0.05, le=0.95, allow_inf_nan=False)
+    signals: tuple[Signal, ...] = DEFAULT_SIGNALS
+
+    @model_validator(mode="after")
+    def validate_signals(self) -> "SignalConfig":
+        if len(self.signals) > 20:
+            raise ValueError("A maximum of 20 signals is allowed")
+        if len(frozenset(signal.id for signal in self.signals)) != len(self.signals):
+            raise ValueError("Signal IDs must be unique")
+        return self
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.model) and bool(self.signals)
+
+    def key(self) -> str:
+        payload: Final = json.dumps(
+            {
+                "model": self.model,
+                "signals": tuple({"id": signal.id, "question": signal.question} for signal in self.signals),
+            },
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+        return hashlib.sha256(payload.encode()).hexdigest()
+
+
+Score: TypeAlias = Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)]
+
+
+class SignalFlag(Record):
+    signal_id: str
+    name: str
+    score: Score
+
+
+class TraceSignals(TraceIdentity):
+    status: Literal["unclassified", "pending", "classified", "failed"]
+    flags: tuple[SignalFlag, ...] = ()
+    model: str = ""
+    classified_at: datetime | None = None
+
+
+class SignalStep(Record):
+    kind: str
+    name: str
+    content: str
+
+
+class SignalData(Record):
+    model_config = ConfigDict(extra="ignore")
+
+    status: Literal["pending", "classified", "failed"] = "pending"
+    scores: Mapping[str, Score] = Field(default_factory=lambda: MappingProxyType({}))
+    model: str = ""
+    error: str = ""
+
+
+class SignalAttempt(Record):
+    status: Literal["classified", "failed"]
+    scores: Mapping[str, Score] = Field(default_factory=lambda: MappingProxyType({}))
+    model: str
+    error: str = ""
+
+
+class StoredTraceSignal(Record):
+    trace_id: str
+    trace_ref: str = ""
+    config_key: str
+    span_count: int
+    claimed_until: datetime | None = None
+    classified_at: datetime | None = None
+    data: JsonValue
+
+    @field_validator("claimed_until", "classified_at")
+    @classmethod
+    def normalize_database_timestamp(cls, value: datetime | None) -> datetime | None:
+        if value is not None and value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value
+
+
+class NoulAnswer(Record):
+    model_config = ConfigDict(extra="ignore", allow_inf_nan=False, from_attributes=True)
+
+    type: Literal["noul"]
+    noul: float = Field(ge=0, le=1, allow_inf_nan=False)
+
+
+class DecisionsOutput(Record):
+    model_config = ConfigDict(extra="ignore", from_attributes=True)
+
+    answers: Mapping[str, object]
+
+
+DecisionState: TypeAlias = Mapping[str, object]
+DecisionQuestions: TypeAlias = Mapping[str, Mapping[str, str]]
+Clock: TypeAlias = Callable[[], datetime]
+RouterReady: TypeAlias = Callable[[], bool]
+
+
+class DecisionsCall(Protocol):
+    async def __call__(
+        self,
+        *,
+        model: str,
+        state: DecisionState,
+        questions: DecisionQuestions,
+        timeout: float,
+        metadata: Mapping[str, object],
+    ) -> object: ...
+
+
+class SignalRepositoryProtocol(Protocol):
+    async def get_config(self) -> SignalConfig: ...
+    async def traces(self, identities: tuple[TraceIdentity, ...]) -> tuple[StoredTraceSignal, ...]: ...
+    async def claim(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        now: datetime,
+    ) -> bool: ...
+    async def store(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        classified_at: datetime,
+        attempt: SignalAttempt,
+    ) -> None: ...
+
+
+def signal_identity(trace: TraceIdentity | StoredTraceSignal | Execution) -> tuple[str, str]:
+    return trace.trace_id, trace.trace_ref
+
+
+def candidate(
+    trace: Execution,
+    existing: StoredTraceSignal | None,
+    config_key: str,
+    now: datetime,
+) -> bool:
+    if existing is None:
+        return True
+    if existing.claimed_until is not None and existing.claimed_until > now:
+        return False
+    if existing.config_key != config_key:
+        return True
+    status: Final = existing.data.get("status") if isinstance(existing.data, dict) else ""
+    if status == "pending":
+        return existing.claimed_until is not None and existing.claimed_until <= now
+    if existing.span_count > trace.span_count:
+        return False
+    if existing.span_count < trace.span_count:
+        return existing.classified_at is not None and existing.classified_at < now - SIGNAL_RECLASSIFY_AFTER
+    return (
+        status == "failed"
+        and existing.classified_at is not None
+        and existing.classified_at < now - SIGNAL_RETRY_FAILED_AFTER
+    )
+
+
+def _take_head(steps: tuple[SignalStep, ...], remaining: int) -> tuple[SignalStep, ...]:
+    if remaining <= 0:
+        return ()
+    cumulative_lengths: Final = tuple(accumulate(len(step.content) for step in steps))
+    boundary: Final = next((index for index, total in enumerate(cumulative_lengths) if total >= remaining), None)
+    if boundary is None:
+        return steps
+    preceding: Final = steps[:boundary]
+    last: Final = steps[boundary]
+    used: Final = cumulative_lengths[boundary - 1] if boundary > 0 else 0
+    last_length: Final = remaining - used
+    return (
+        *preceding,
+        last
+        if last_length == len(last.content)
+        else last.model_copy(update=MappingProxyType({"content": last.content[:last_length]})),
+    )
+
+
+def _take_tail(steps: tuple[SignalStep, ...], remaining: int) -> tuple[SignalStep, ...]:
+    if remaining <= 0:
+        return ()
+    reversed_steps: Final = tuple(reversed(steps))
+    cumulative_lengths: Final = tuple(accumulate(len(step.content) for step in reversed_steps))
+    boundary: Final = next((index for index, total in enumerate(cumulative_lengths) if total >= remaining), None)
+    if boundary is None:
+        return steps
+    preceding: Final = reversed_steps[:boundary]
+    last: Final = reversed_steps[boundary]
+    used: Final = cumulative_lengths[boundary - 1] if boundary > 0 else 0
+    last_length: Final = remaining - used
+    selected: Final = (
+        *preceding,
+        last
+        if last_length == len(last.content)
+        else last.model_copy(update=MappingProxyType({"content": last.content[-last_length:]})),
+    )
+    return tuple(reversed(selected))
+
+
+def _bounded_steps(steps: tuple[SignalStep, ...]) -> tuple[SignalStep, ...]:
+    if sum(len(step.content) for step in steps) <= SIGNAL_TRANSCRIPT_MAX_CHARS:
+        return steps
+    head: Final = _take_head(steps, SIGNAL_TRANSCRIPT_HEAD_CHARS)
+    tail: Final = _take_tail(steps, SIGNAL_TRANSCRIPT_TAIL_CHARS)
+    omitted_count: Final = len(steps) - len(head) - len(tail)
+    marker: Final = SignalStep(kind="omitted", name="", content=f"{omitted_count} steps omitted")
+    return (*head, marker, *tail)
+
+
+def _part_excerpt(content: str) -> str:
+    if len(content) <= SIGNAL_PART_MAX_CHARS:
+        return content
+    omitted: Final = len(content) - SIGNAL_PART_MAX_CHARS
+    marker: Final = f"\n[... {omitted} characters omitted ...]\n"
+    return f"{content[:SIGNAL_PART_HEAD_CHARS]}{marker}{content[-SIGNAL_PART_TAIL_CHARS:]}"
+
+
+async def _content_pages(
+    reader: SourceReader,
+    scope: Scope,
+    execution: Execution,
+    cursor: str,
+    pages_left: int,
+) -> tuple[SignalStep, ...]:
+    if pages_left == 0:
+        return ()
+    content: Final = await reader.content(scope, execution, cursor)
+    current: Final = tuple(
+        SignalStep(kind=part.kind, name=part.name, content=_part_excerpt(part.content)) for part in content.parts
+    )
+    rest: Final = (
+        await _content_pages(reader, scope, execution, content.next_cursor, pages_left - 1)
+        if content.next_cursor is not None
+        else ()
+    )
+    return (*current, *rest)
+
+
+async def signal_state(reader: SourceReader, scope: Scope, execution: Execution) -> DecisionState:
+    steps: Final = _bounded_steps(await _content_pages(reader, scope, execution, "", SIGNAL_MAX_CONTENT_PAGES))
+    return {
+        "task": SIGNAL_TASK,
+        "steps": tuple(step.model_dump(mode="json") for step in steps),
+    }
+
+
+def _noul_score(value: object) -> float | None:
+    try:
+        return NoulAnswer.model_validate(value).noul
+    except ValidationError:
+        return None
+
+
+class SignalClassifier:
+    def __init__(self, reader: SourceReader, completion: DecisionsCall, clock: Clock) -> None:
+        self.reader: Final = reader
+        self.completion: Final = completion
+        self.clock: Final = clock
+
+    async def classify(self, scope: Scope, execution: Execution, config: SignalConfig) -> SignalAttempt:
+        try:
+            state: Final = await signal_state(self.reader, scope, execution)
+            questions: Final = {
+                signal.id: {"type": "noul", "instructions": signal.question} for signal in config.signals
+            }
+            with lens_analysis(), inherit_message_logging_privacy(True):
+                response: Final = await self.completion(
+                    model=config.model,
+                    state=state,
+                    questions=questions,
+                    timeout=60,
+                    metadata={"tags": ["litellm-lens-signals"]},
+                )
+            output: Final = DecisionsOutput.model_validate(response)
+            scores: Final = MappingProxyType(
+                {
+                    signal.id: score
+                    for signal in config.signals
+                    if (score := _noul_score(output.answers.get(signal.id))) is not None
+                }
+            )
+            if len(scores) != len(config.signals):
+                return SignalAttempt(
+                    status="failed",
+                    scores=scores,
+                    model=config.model,
+                    error="Decisions response omitted a configured noul answer",
+                )
+            return SignalAttempt(status="classified", scores=scores, model=config.model)
+        except Exception as error:
+            detail: Final = redact_internal_details(str(error))[:300]
+            return SignalAttempt(status="failed", model=config.model, error=detail)
+
+
+def trace_signals(
+    trace: TraceIdentity,
+    existing: StoredTraceSignal | None,
+    config: SignalConfig,
+) -> TraceSignals:
+    if existing is None or existing.config_key != config.key():
+        return TraceSignals(trace_id=trace.trace_id, trace_ref=trace.trace_ref, status="unclassified")
+    data: Final = SignalData.model_validate(existing.data)
+    if data.status == "pending":
+        return TraceSignals(
+            trace_id=trace.trace_id,
+            trace_ref=trace.trace_ref,
+            status="pending",
+            model=data.model,
+        )
+    if data.status == "failed" or data.error:
+        return TraceSignals(
+            trace_id=trace.trace_id,
+            trace_ref=trace.trace_ref,
+            status="failed",
+            model=data.model,
+            classified_at=existing.classified_at,
+        )
+    flags: Final = tuple(
+        sorted(
+            (
+                SignalFlag(signal_id=signal.id, name=signal.name, score=data.scores[signal.id])
+                for signal in config.signals
+                if signal.id in data.scores and data.scores[signal.id] >= config.threshold
+            ),
+            key=lambda flag: flag.score,
+            reverse=True,
+        )
+    )
+    return TraceSignals(
+        trace_id=trace.trace_id,
+        trace_ref=trace.trace_ref,
+        status="classified",
+        flags=flags,
+        model=data.model,
+        classified_at=existing.classified_at,
+    )
+
+
+async def _process_claimed(
+    classifier: SignalClassifier,
+    repository: SignalRepositoryProtocol,
+    scope: Scope,
+    execution: Execution,
+    config: SignalConfig,
+    claimed_until: datetime,
+) -> None:
+    from litellm._logging import verbose_proxy_logger
+
+    attempt: Final = await classifier.classify(scope, execution, config)
+    try:
+        await repository.store(execution, config, claimed_until, classifier.clock(), attempt)
+    except Exception as error:
+        verbose_proxy_logger.error("Lens signal result could not be stored: %s", redact_internal_details(str(error)))
+
+
+class _SignalScan:
+    def __init__(
+        self,
+        reader: SourceReader,
+        repository: SignalRepositoryProtocol,
+        scope: Scope,
+        config: SignalConfig,
+        now: datetime,
+        cursor: str,
+        limit: int,
+    ) -> None:
+        self.reader: Final = reader
+        self.repository: Final = repository
+        self.scope: Final = scope
+        self.config: Final = config
+        self.now: Final = now
+        self.cursor: str = cursor
+        self.limit: Final = limit
+        self.executions: tuple[Execution, ...] = ()
+        self.finished: bool = False
+
+    async def _read_page(self, start: int, end: int) -> tuple[tuple[Execution, ...], str | None]:
+        page_cursor: Final = self.cursor
+        sample: Final = await self.reader.sample(
+            self.scope,
+            ActivitySelection(source="traces"),
+            start,
+            end,
+            page_size=SIGNAL_PAGE_SIZE,
+            cursor=page_cursor,
+        )
+        identities: Final = tuple(
+            TraceIdentity(trace_id=trace.trace_id, trace_ref=trace.trace_ref) for trace in sample.executions
+        )
+        existing_rows: Final = await self.repository.traces(identities)
+        existing: Final = MappingProxyType({signal_identity(row): row for row in existing_rows})
+        remaining: Final = self.limit - len(self.executions)
+        all_eligible: Final = tuple(
+            execution
+            for execution in sample.executions
+            if candidate(execution, existing.get(signal_identity(execution)), self.config.key(), self.now)
+        )
+        eligible: Final = all_eligible[:remaining]
+        next_cursor: Final = page_cursor if len(all_eligible) > remaining else sample.next_cursor
+        return eligible, next_cursor
+
+    async def run(self) -> tuple[tuple[Execution, ...], str]:
+        start: Final = int((self.now - timedelta(hours=24)).timestamp() * 1000)
+        end: Final = int((self.now - timedelta(minutes=2)).timestamp() * 1000)
+        for _ in range(SIGNAL_MAX_SCAN_PAGES):
+            if self.finished or len(self.executions) >= self.limit:
+                break
+            eligible, next_cursor = await self._read_page(start, end)
+            self.executions = (*self.executions, *eligible)
+            if next_cursor is None:
+                self.cursor = ""
+                self.finished = True
+            else:
+                self.cursor = next_cursor
+        return self.executions, self.cursor
+
+
+async def _scan_pages(
+    reader: SourceReader,
+    repository: SignalRepositoryProtocol,
+    scope: Scope,
+    config: SignalConfig,
+    now: datetime,
+    cursor: str,
+    remaining: int,
+) -> tuple[tuple[Execution, ...], str]:
+    if remaining <= 0:
+        return (), cursor
+    scan: Final = _SignalScan(reader, repository, scope, config, now, cursor, remaining)
+    return await scan.run()
+
+
+async def run_signal_tick(
+    storage: Storage,
+    repository: SignalRepositoryProtocol | None,
+    completion: DecisionsCall | None,
+    clock: Clock,
+    router_ready: RouterReady = lambda: True,
+    cursor: str = "",
+) -> str:
+    if repository is None or completion is None or not router_ready():
+        return cursor
+    now: Final = clock()
+    config: Final = await repository.get_config()
+    if not config.enabled:
+        return cursor
+    reader: Final = SourceReader(storage)
+    scope: Final = Scope(all_teams=True)
+    candidates: Final = await _scan_pages(
+        reader,
+        repository,
+        scope,
+        config,
+        now,
+        cursor,
+        SIGNAL_MAX_PER_TICK,
+    )
+    executions, next_cursor = candidates
+    classifier: Final = SignalClassifier(reader, completion, clock)
+    semaphore: Final = asyncio.Semaphore(SIGNAL_CONCURRENCY)
+
+    async def process(execution: Execution) -> None:
+        from litellm._logging import verbose_proxy_logger
+
+        async with semaphore:
+            claimed_at: Final = classifier.clock()
+            claimed_until: Final = claimed_at + SIGNAL_CLAIM_LEASE
+            try:
+                claimed: Final = await repository.claim(execution, config, claimed_until, claimed_at)
+            except Exception as error:
+                verbose_proxy_logger.error("Lens signal claim failed: %s", redact_internal_details(str(error)))
+                return
+            if not claimed:
+                return
+            await _process_claimed(classifier, repository, scope, execution, config, claimed_until)
+
+    await asyncio.gather(*(process(execution) for execution in executions))
+    return next_cursor
+
+
+class _SignalLoopState:
+    def __init__(self) -> None:
+        self.cursor: str = ""
+
+
+async def run_signal_loop(
+    storage: Storage,
+    repository: SignalRepositoryProtocol | None,
+    completion: DecisionsCall | None,
+    clock: Clock = lambda: datetime.now(timezone.utc),
+    router_ready: RouterReady = lambda: True,
+) -> None:
+    from litellm._logging import verbose_proxy_logger
+
+    state: Final = _SignalLoopState()
+    while True:
+        try:
+            state.cursor = await run_signal_tick(
+                storage,
+                repository,
+                completion,
+                clock,
+                router_ready,
+                cursor=state.cursor,
+            )
+        except Exception as error:
+            verbose_proxy_logger.error("Lens signal tick failed: %s", redact_internal_details(str(error)))
+        await asyncio.sleep(SIGNAL_INTERVAL_SECONDS)

litellm/proxy/lens/sources.py

@@ -142,6 +142,7 @@ class SourceReader:
             id=execution.trace_id,
             trace_ref=execution.trace_ref,
             record_team=execution.team_id,
+            start_time=execution.start_time,
             cursor=cursor,
             offset=offset + 1,
         )
@@ -175,6 +176,7 @@ class SourceReader:
             id=execution.trace_id,
             trace_ref=execution.trace_ref,
             record_team=execution.team_id,
+            start_time=execution.start_time,
             span=evidence.span_id,
             quote=evidence.quote,
         )

litellm/proxy/proxy_server.py

@@ -581,6 +581,14 @@ from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_sp
 from litellm.proxy.image_endpoints.endpoints import router as image_router
 from litellm.proxy.lens.dataset_endpoints import router as lens_dataset_router
 from litellm.proxy.lens.endpoints import router as lens_router
+from litellm.proxy.lens.repository import WriterDatabase
+from litellm.proxy.lens.signal_repository import SignalRepository
+from litellm.proxy.lens.signals import (
+    DecisionQuestions,
+    DecisionsCall,
+    DecisionState,
+    run_signal_loop,
+)
 from litellm.proxy.list_api.common import (
     ManagementProblem,
     problem_response,
@@ -1275,6 +1283,27 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries:
     return client.writer_db
 
 
+async def _call_current_lens_signal_router(
+    *,
+    model: str,
+    state: DecisionState,
+    questions: DecisionQuestions,
+    timeout: float,
+    metadata: Mapping[str, object],
+) -> object:
+    current_router: Final = llm_router
+    if current_router is None:
+        raise RuntimeError("The proxy router is not initialized")
+    decisions: Final[DecisionsCall] = cast(DecisionsCall, current_router.adecisions)
+    return await decisions(
+        model=model,
+        state=state,
+        questions=questions,
+        timeout=timeout,
+        metadata=metadata,
+    )
+
+
 @asynccontextmanager
 async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]:
     global \
@@ -1645,12 +1674,30 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
         state: Final[ProxyLifespanState] = {"tracing_receiver": receiver}
         from litellm.proxy.admin_mcp import admin_mcp_lifespan
 
+        signal_completion: Final[DecisionsCall] = _call_current_lens_signal_router
+        signal_task: Final = (
+            asyncio.create_task(
+                run_signal_loop(
+                    receiver.storage,
+                    SignalRepository(WriterDatabase(writer_wrapper(prisma_client.db))),
+                    signal_completion,
+                    router_ready=lambda: llm_router is not None,
+                )
+            )
+            if receiver is not None and prisma_client is not None
+            else None
+        )
+
         try:
             async with AsyncExitStack() as admin_mcp_stack:
                 try:
                     await admin_mcp_stack.enter_async_context(admin_mcp_lifespan(app))
                     yield state
                 finally:
+                    if signal_task is not None:
+                        signal_task.cancel()
+                        await asyncio.gather(signal_task, return_exceptions=True)
+
                     if model_info_scheduler is not None and model_info_scheduler.running:
                         model_info_scheduler.remove_job("refresh_model_info")
                         if model_info_scheduler is not scheduler:
@@ -13951,7 +13998,7 @@ async def run_thread(
 # )
 # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)):
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
+from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.model_repository import ModelRepository
 from litellm.repositories.table_repositories import (

litellm/proxy/schema.prisma

@@ -1982,3 +1982,20 @@ model LiteLLM_LensDataset {
 
   @@id([id, revision])
 }
+
+model LiteLLM_LensSignalConfig {
+  id   String @id
+  data Json
+}
+
+model LiteLLM_LensTraceSignal {
+  trace_id      String
+  trace_ref     String @default("")
+  config_key    String
+  span_count    Int
+  claimed_until DateTime?
+  classified_at DateTime?
+  data          Json
+
+  @@id([trace_id, trace_ref])
+}

litellm/rust_bridge/trace/generated/models.py

@@ -215,6 +215,7 @@ class LensContentParams(LiteLLMBaseModel):
     source: ContentSource
     id: str
     record_team: str
+    start_time: str
     trace_ref: str
     cursor: str
     offset: int = Field(..., ge=0, le=4294967295)
@@ -232,6 +233,7 @@ class LensEvidenceParams(LiteLLMBaseModel):
     source: ContentSource
     id: str
     record_team: str
+    start_time: str
     trace_ref: str
     span: str
     quote: str

schema.prisma

@@ -1982,3 +1982,20 @@ model LiteLLM_LensDataset {
 
   @@id([id, revision])
 }
+
+model LiteLLM_LensSignalConfig {
+  id   String @id
+  data Json
+}
+
+model LiteLLM_LensTraceSignal {
+  trace_id      String
+  trace_ref     String @default("")
+  config_key    String
+  span_count    Int
+  claimed_until DateTime?
+  classified_at DateTime?
+  data          Json
+
+  @@id([trace_id, trace_ref])
+}

scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json

@@ -39,6 +39,9 @@
     "source": {
       "$ref": "#/$defs/ContentSource"
     },
+    "start_time": {
+      "type": "string"
+    },
     "team": {
       "type": "string"
     },
@@ -53,6 +56,7 @@
     "source",
     "id",
     "record_team",
+    "start_time",
     "trace_ref",
     "cursor",
     "offset"

scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json

@@ -36,6 +36,9 @@
     "span": {
       "type": "string"
     },
+    "start_time": {
+      "type": "string"
+    },
     "team": {
       "type": "string"
     },
@@ -50,6 +53,7 @@
     "source",
     "id",
     "record_team",
+    "start_time",
     "trace_ref",
     "span",
     "quote"

tests/e2e/AGENTS.md

@@ -215,7 +215,7 @@ llm.<endpoint>.<route>.<capability>.<streaming>.<assertion>
                | rerank | images_generations | audio_speech | audio_transcriptions | moderations
                | realtime
   route      : openai | azure_openai | anthropic | bedrock_converse | bedrock_invoke | vertex
-               | azure_foundry | cohere | together_ai
+               | azure_foundry | cohere | together_ai | ollama | ollama_chat
                (vocab varies per endpoint; messages is anthropic-format only)
   capability : basic | tool_use | prompt_cache_5m | vision | thinking | structured_output
                | service_tier | mid_conversation_system

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -101,6 +101,32 @@
 - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"}
 - {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"}
 - {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"}
+- {id: llm.chat_completions.ollama_chat.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama_chat, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /chat/completions returns text and usage"}
+- {id: llm.chat_completions.ollama_chat.basic.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama_chat, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /chat/completions streams text deltas, usage and a terminal event"}
+- {id: llm.chat_completions.ollama_chat.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama_chat, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /chat/completions returns one addressable get_weather call"}
+- {id: llm.chat_completions.ollama_chat.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama_chat, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /chat/completions tool result round trip reaches the model"}
+- {id: llm.chat_completions.ollama_chat.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama_chat, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) streams the call as tool_call deltas ending in finish_reason tool_calls, the shape coding agents like OpenCode execute"}
+- {id: llm.messages.ollama_chat.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama_chat, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/messages returns text and usage"}
+- {id: llm.messages.ollama_chat.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama_chat, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/messages streams text deltas, usage and a terminal event"}
+- {id: llm.messages.ollama_chat.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama_chat, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/messages returns one addressable get_weather call"}
+- {id: llm.messages.ollama_chat.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama_chat, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/messages tool result round trip reaches the model"}
+- {id: llm.responses.ollama_chat.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama_chat, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/responses returns text and usage"}
+- {id: llm.responses.ollama_chat.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama_chat, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/responses streams text deltas, usage and a terminal event"}
+- {id: llm.responses.ollama_chat.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama_chat, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/responses returns one addressable get_weather call"}
+- {id: llm.responses.ollama_chat.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama_chat, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/chat (native tools) over /v1/responses tool result round trip reaches the model"}
+- {id: llm.chat_completions.ollama.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /chat/completions returns text and usage"}
+- {id: llm.chat_completions.ollama.basic.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /chat/completions streams text deltas, usage and a terminal event"}
+- {id: llm.chat_completions.ollama.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /chat/completions returns one addressable get_weather call"}
+- {id: llm.chat_completions.ollama.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /chat/completions tool result round trip reaches the model"}
+- {id: llm.chat_completions.ollama.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: ollama, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) streams the call as tool_call deltas ending in finish_reason tool_calls, the shape coding agents like OpenCode execute; prompt-based JSON used to arrive as plain text with finish_reason stop (GitHub issue #35711)"}
+- {id: llm.messages.ollama.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/messages returns text and usage"}
+- {id: llm.messages.ollama.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/messages streams text deltas, usage and a terminal event"}
+- {id: llm.messages.ollama.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/messages returns one addressable get_weather call"}
+- {id: llm.messages.ollama.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: ollama, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/messages tool result round trip reaches the model"}
+- {id: llm.responses.ollama.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses returns text and usage"}
+- {id: llm.responses.ollama.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses streams text deltas, usage and a terminal event"}
+- {id: llm.responses.ollama.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses returns one addressable get_weather call"}
+- {id: llm.responses.ollama.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses tool result round trip reaches the model"}
 - {id: llm.chat_completions.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "service_tier balanced and auto map to Sail completion windows and bill the matching price columns; Sail serves flex only to background responses and Batch"}
 - {id: llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [rejects_unknown_tier], source: "llm_translation/test_sail_e2e.py", rationale: "A service_tier Sail has no completion window for is a 400 without drop_params"}
 - {id: llm.chat_completions.sail.service_tier.nonstream.drops_unknown_tier_and_bills_asap, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [drops_unknown_tier_and_bills_asap], source: "llm_translation/test_sail_e2e.py", rationale: "An unknown service_tier under drop_params is dropped and billed at asap in both the cost header and spend log"}

tests/e2e/coverage_registry/schema.py

@@ -55,6 +55,8 @@ LlmRoute = Literal[
     "cohere",
     "gemini",
     "hosted_vllm",
+    "ollama",
+    "ollama_chat",
     "openai",
     "sail",
     "together_ai",

tests/e2e/llm_translation/test_ollama_e2e.py

@@ -0,0 +1,258 @@
+"""Ollama behind the proxy on /chat/completions, /v1/messages and /v1/responses.
+
+Ollama has two litellm routes with different tool plumbing: `ollama_chat/` calls
+/api/chat and forwards native tools, while `ollama/` calls /api/generate, which
+has no tools field, so litellm prompts the model for a JSON function call and
+turns that JSON back into a tool call. Each route runs the same conversation
+contract as the conversational matrix on every surface, through the matrix's
+SDK-backed surfaces, plus a streamed tool call on chat completions, the shape
+coding agents such as OpenCode consume.
+
+The deployments set drop_params because Ollama has no parallel_tool_calls,
+which the matrix surfaces send alongside a forced tool_choice. Live only: Ollama
+Cloud has no provider edge mount, and its requests are not priced in the cost
+map, so there is no cost cell here.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from itertools import chain, product
+from types import MappingProxyType
+from typing import Final, Literal, cast
+
+import pytest
+from _pytest.mark.structures import ParameterSet
+from e2e_config import unique_marker
+from e2e_metadata import Capability as SubjectCapability
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
+from lifecycle import ResourceManager
+from llm_translation.conversational_matrix import (
+    GREETING_PROMPT,
+    INSTRUCTIONS,
+    MAX_OUTPUT_TOKENS,
+    SURFACES,
+    WEATHER_PROMPT,
+    WEATHER_REPORT,
+    WEATHER_TOOL_DESCRIPTION,
+    WEATHER_TOOL_NAME,
+    WEATHER_TOOL_SCHEMA,
+    Surface,
+    SurfaceName,
+    ToolCall,
+    WeatherArgs,
+    build_surfaces,
+)
+from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients
+from models import LiteLLMParamsBody
+from openai.types.chat import ChatCompletionChunk, ChatCompletionToolParam
+from proxy_client import ProxyClient
+
+pytestmark = pytest.mark.e2e
+
+OllamaRoute = Literal["ollama_chat", "ollama"]
+Capability = Literal["basic", "tool_use", "multi_turn"]
+Streaming = Literal["stream", "nonstream"]
+
+OLLAMA_API_BASE: Final = "https://ollama.com"
+OLLAMA_MODEL: Final = "gemma4:31b"
+ROUTES: Final[tuple[OllamaRoute, ...]] = ("ollama_chat", "ollama")
+
+
+@dataclass(frozen=True, slots=True)
+class Cell:
+    surface: SurfaceName
+    route: OllamaRoute
+
+    @property
+    def id(self) -> str:
+        return f"{self.surface}-{self.route}"
+
+
+def _cells(capability: Capability, streaming: Streaming) -> tuple[ParameterSet, ...]:
+    return tuple(
+        pytest.param(
+            Cell(surface=surface, route=route),
+            id=f"{surface}-{route}",
+            marks=pytest.mark.covers(f"llm.{surface}.{route}.{capability}.{streaming}.works"),
+        )
+        for surface, route in product(SURFACES, ROUTES)
+    )
+
+
+def _streamed_tool_cells() -> tuple[ParameterSet, ...]:
+    return tuple(
+        pytest.param(route, id=route, marks=pytest.mark.covers(f"llm.chat_completions.{route}.tool_use.stream.works"))
+        for route in ROUTES
+    )
+
+
+def _register(proxy: ProxyClient, resources: ResourceManager, route: OllamaRoute) -> str:
+    alias: Final = f"e2e-ollama-{route}-{unique_marker()}"
+    model_id: Final = proxy.create_model(
+        alias,
+        LiteLLMParamsBody(
+            model=f"{route}/{OLLAMA_MODEL}",
+            api_base=OLLAMA_API_BASE,
+            api_key="os.environ/OLLAMA_API_KEY",
+            drop_params=True,
+        ),
+    )
+    resources.defer(lambda: proxy.delete_model(model_id))
+    return alias
+
+
+@pytest.fixture(scope="module")
+def aliases(proxy: ProxyClient) -> Iterator[Mapping[OllamaRoute, str]]:
+    resources: Final = ResourceManager(client=proxy)
+    try:
+        yield MappingProxyType({route: _register(proxy, resources, route) for route in ROUTES})
+    finally:
+        resources.teardown()
+
+
+@pytest.fixture(scope="module")
+def surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]:
+    return build_surfaces(sdk)
+
+
+def _weather_call(cell: Cell, surface: Surface, key: str, model: str) -> ToolCall:
+    reply: Final = surface.reply(key, model, WEATHER_PROMPT, with_tool=True)
+    assert len(reply.tool_calls) == 1, (
+        f"{cell.id}: expected one {WEATHER_TOOL_NAME} call, got {reply.tool_calls} text={reply.text!r}"
+    )
+    call: Final = reply.tool_calls[0]
+    assert call.name == WEATHER_TOOL_NAME, f"{cell.id}: called {call.name!r}, not {WEATHER_TOOL_NAME!r}"
+    assert call.call_id, f"{cell.id}: tool call has no id, so the caller cannot answer it: {call}"
+    assert "paris" in call.parsed().location.lower(), f"{cell.id}: tool arguments lost the location: {call}"
+    return call
+
+
+def _weather_tool() -> ChatCompletionToolParam:
+    return {
+        "type": "function",
+        "function": {
+            "name": WEATHER_TOOL_NAME,
+            "description": WEATHER_TOOL_DESCRIPTION,
+            "parameters": dict(WEATHER_TOOL_SCHEMA),
+        },
+    }
+
+
+def _subject(mode: Mode, *, tools: bool, route: Route | None = None) -> Subject:
+    return Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=route,
+        providers=(Provider.OLLAMA,),
+        models=(OLLAMA_MODEL,),
+        capabilities=(SubjectCapability.FUNCTION_CALLING,) if tools else (),
+        mode=mode,
+    )
+
+
+class TestOllamaConversation:
+    @pytest.mark.parametrize("cell", _cells("basic", "nonstream"))
+    @meta(_subject(Mode.NONSTREAM, tools=False))
+    def test_reply_carries_assistant_text_and_usage(
+        self,
+        cell: Cell,
+        aliases: Mapping[OllamaRoute, str],
+        surfaces: Mapping[SurfaceName, Surface],
+        resources: ResourceManager,
+    ) -> None:
+        reply: Final = surfaces[cell.surface].reply(resources.key(), aliases[cell.route], GREETING_PROMPT)
+
+        assert reply.response_id, f"{cell.id}: response has no id"
+        assert reply.text.strip(), f"{cell.id}: response carried no assistant text"
+        assert reply.usage is not None and reply.usage.input_tokens > 0 and reply.usage.output_tokens > 0, (
+            f"{cell.id}: usage missing or zero: {reply.usage}"
+        )
+        assert reply.call_id_header, f"{cell.id}: x-litellm-call-id header missing"
+
+    @pytest.mark.parametrize("cell", _cells("basic", "stream"))
+    @meta(_subject(Mode.STREAM, tools=False))
+    def test_stream_delivers_text_usage_and_a_terminal_event(
+        self,
+        cell: Cell,
+        aliases: Mapping[OllamaRoute, str],
+        surfaces: Mapping[SurfaceName, Surface],
+        resources: ResourceManager,
+    ) -> None:
+        streamed: Final = surfaces[cell.surface].stream(resources.key(), aliases[cell.route], GREETING_PROMPT)
+
+        assert streamed.event_count > 1, f"{cell.id}: stream arrived as {streamed.event_count} event(s)"
+        assert streamed.text.strip(), f"{cell.id}: stream carried no text deltas"
+        assert streamed.finished, f"{cell.id}: stream never sent its terminal event"
+        assert streamed.usage_reported, f"{cell.id}: stream never reported usage"
+
+    @pytest.mark.parametrize("cell", _cells("tool_use", "nonstream"))
+    @meta(_subject(Mode.NONSTREAM, tools=True))
+    def test_tool_call_is_returned_named_and_addressable(
+        self,
+        cell: Cell,
+        aliases: Mapping[OllamaRoute, str],
+        surfaces: Mapping[SurfaceName, Surface],
+        resources: ResourceManager,
+    ) -> None:
+        _ = _weather_call(cell, surfaces[cell.surface], resources.key(), aliases[cell.route])
+
+    @pytest.mark.parametrize("cell", _cells("multi_turn", "nonstream"))
+    @meta(_subject(Mode.NONSTREAM, tools=True))
+    def test_tool_result_round_trip_reaches_the_model(
+        self,
+        cell: Cell,
+        aliases: Mapping[OllamaRoute, str],
+        surfaces: Mapping[SurfaceName, Surface],
+        resources: ResourceManager,
+    ) -> None:
+        key: Final = resources.key()
+        model: Final = aliases[cell.route]
+        surface: Final = surfaces[cell.surface]
+        call: Final = _weather_call(cell, surface, key, model)
+
+        answer: Final = surface.reply_to_tool_result(key, model, WEATHER_PROMPT, call, WEATHER_REPORT)
+        assert "22" in answer.text, f"{cell.id}: the model never saw the tool result: {answer.text!r}"
+
+
+class TestOllamaStreamedToolCall:
+    @pytest.mark.parametrize("route", _streamed_tool_cells())
+    @meta(_subject(Mode.STREAM, tools=True, route=Route.CHAT_COMPLETIONS))
+    def test_tool_call_streams_as_tool_call_deltas(
+        self,
+        route: OllamaRoute,
+        aliases: Mapping[OllamaRoute, str],
+        sdk: SdkClients,
+        resources: ResourceManager,
+    ) -> None:
+        chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple(
+            sdk.openai(resources.key()).chat.completions.create(
+                model=aliases[route],
+                messages=[
+                    {"role": "system", "content": INSTRUCTIONS},
+                    {"role": "user", "content": WEATHER_PROMPT},
+                ],
+                tools=[_weather_tool()],
+                max_completion_tokens=MAX_OUTPUT_TOKENS,
+                stream=True,
+                extra_body=NO_PROXY_CACHE,
+            )
+        )
+        choices: Final = tuple(chunk.choices[0] for chunk in chunks if chunk.choices)
+        text: Final = "".join(choice.delta.content or "" for choice in choices)
+        deltas: Final = tuple(chain.from_iterable(choice.delta.tool_calls or () for choice in choices))
+        call_ids: Final = tuple(delta.id for delta in deltas if delta.id)
+        indexes: Final = frozenset(delta.index for delta in deltas)
+        functions: Final = tuple(delta.function for delta in deltas if delta.function is not None)
+        names: Final = tuple(function.name for function in functions if function.name)
+        arguments: Final = "".join(function.arguments or "" for function in functions)
+        finish_reasons: Final = tuple(choice.finish_reason for choice in choices if choice.finish_reason is not None)
+
+        assert names == (WEATHER_TOOL_NAME,), f"{route}: streamed tool names {names}, text={text!r}"
+        assert len(call_ids) == 1, f"{route}: expected one streamed tool call id, got {call_ids}"
+        assert indexes == {0}, f"{route}: streamed tool call deltas used indexes {sorted(indexes)}"
+        assert WEATHER_TOOL_NAME not in text, f"{route}: the tool call leaked into assistant text: {text!r}"
+        location: Final = WeatherArgs.model_validate(cast(object, json.loads(arguments))).location
+        assert "paris" in location.lower(), f"{route}: streamed tool arguments lost the location: {arguments!r}"
+        assert finish_reasons[-1:] == ("tool_calls",), f"{route}: stream finished with {finish_reasons}"

tests/e2e/models.py

@@ -1257,6 +1257,7 @@ class LiteLLMParamsBody(BaseModel):
     api_version: str | None = None
     realtime_protocol: str | None = None
     allowed_openai_params: list[str] | None = None
+    drop_params: bool | None = None
     aws_access_key_id: str | None = Field(default=None, repr=False)
     aws_secret_access_key: str | None = Field(default=None, repr=False)
     aws_region_name: str | None = None

tests/integration/providers/test_ollama_prompt_tools_chaos.py

@@ -0,0 +1,404 @@
+import asyncio
+import json
+import re
+import signal
+import threading
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from queue import SimpleQueue
+from types import MappingProxyType
+from typing import Final, Literal
+from urllib.parse import urlsplit
+
+import httpx
+import psutil
+import pytest
+import yaml
+from integration._support.client import Gateway, eventually
+from integration._support.database import read_rows
+from integration._support.process import graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+_BACKEND: Final = "llama3-prompt-tools-chaos"
+_API_KEY: Final = "synthetic-ollama-key"
+_CONFIG_MODEL: Final = "ollama-prompt-tools-chaos"
+_INSTRUCTION: Final = (
+    'To call a function, reply with JSON ONLY in this format {"name": "function_name", '
+    '"arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, '
+    "reply to the user in plain text instead of calling a function again. "
+    "The following functions are available to you:"
+)
+_CALL_ID: Final = "call_prompt_tools_chaos_1"
+_ARGUMENTS: Final[dict[str, JsonValue]] = {"city": "Paris"}
+_PARAMETERS: Final[dict[str, JsonValue]] = {
+    "type": "object",
+    "properties": {"city": {"type": "string"}},
+    "required": ["city"],
+}
+_CHAT_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "function": {"name": "get_weather", "description": "Weather for a city", "parameters": _PARAMETERS},
+}
+_ANTHROPIC_TOOL: Final[dict[str, JsonValue]] = {
+    "name": "get_weather",
+    "description": "Weather for a city",
+    "input_schema": _PARAMETERS,
+}
+_RESPONSES_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "name": "get_weather",
+    "description": "Weather for a city",
+    "parameters": _PARAMETERS,
+}
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
+_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
+_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120
+
+Endpoint = Literal["chat", "messages", "responses"]
+
+
+@dataclass(frozen=True, slots=True)
+class _Call:
+    endpoint: Endpoint
+    stream: bool
+    marker: str
+
+
+@dataclass(frozen=True, slots=True)
+class _Served:
+    call: _Call
+    status: int
+    text: str
+
+
+def _result(marker: str) -> str:
+    return f"Paris: 22 degrees Celsius marker-{marker}"
+
+
+def _answer(marker: str) -> str:
+    return f"answer marker-{marker}"
+
+
+def _path(endpoint: Endpoint) -> str:
+    match endpoint:
+        case "chat":
+            return "/v1/chat/completions"
+        case "messages":
+            return "/v1/messages"
+        case "responses":
+            return "/v1/responses"
+
+
+def _body(model: str, call: _Call) -> dict[str, JsonValue]:
+    question: Final = "What is the weather in Paris?"
+    common: Final[dict[str, JsonValue]] = {
+        "model": model,
+        "stream": call.stream,
+        "num_retries": 0,
+        "cache": {"no-cache": True},
+    }
+    match call.endpoint:
+        case "chat":
+            return {
+                **common,
+                "tools": [_CHAT_TOOL],
+                "messages": [
+                    {"role": "user", "content": question},
+                    {
+                        "role": "assistant",
+                        "content": None,
+                        "tool_calls": [
+                            {
+                                "id": _CALL_ID,
+                                "type": "function",
+                                "function": {"name": "get_weather", "arguments": json.dumps(_ARGUMENTS)},
+                            }
+                        ],
+                    },
+                    {"role": "tool", "tool_call_id": _CALL_ID, "content": _result(call.marker)},
+                ],
+            }
+        case "messages":
+            return {
+                **common,
+                "max_tokens": 64,
+                "tools": [_ANTHROPIC_TOOL],
+                "messages": [
+                    {"role": "user", "content": question},
+                    {
+                        "role": "assistant",
+                        "content": [{"type": "tool_use", "id": _CALL_ID, "name": "get_weather", "input": _ARGUMENTS}],
+                    },
+                    {
+                        "role": "user",
+                        "content": [{"type": "tool_result", "tool_use_id": _CALL_ID, "content": _result(call.marker)}],
+                    },
+                ],
+            }
+        case "responses":
+            return {
+                **common,
+                "store": False,
+                "tools": [_RESPONSES_TOOL],
+                "input": [
+                    {"role": "user", "content": question},
+                    {
+                        "type": "function_call",
+                        "call_id": _CALL_ID,
+                        "name": "get_weather",
+                        "arguments": json.dumps(_ARGUMENTS),
+                    },
+                    {"type": "function_call_output", "call_id": _CALL_ID, "output": _result(call.marker)},
+                ],
+            }
+
+
+def _generate_reply(marker: str, stream: bool, drop_connection: bool = False) -> Reply:
+    done: Final = {
+        "model": _BACKEND,
+        "created_at": "2026-10-07T00:00:00Z",
+        "response": "",
+        "done": True,
+        "done_reason": "stop",
+        "prompt_eval_count": 30,
+        "eval_count": 12,
+    }
+    if not stream:
+        return Reply(body=json.dumps({**done, "response": _answer(marker)}).encode(), drop_connection=drop_connection)
+    pieces: Final = ("answer ", f"marker-{marker}")
+    frames: Final = (
+        *(
+            {"model": _BACKEND, "created_at": "2026-10-07T00:00:00Z", "response": piece, "done": False}
+            for piece in pieces
+        ),
+        done,
+    )
+    return Reply(
+        content_type="application/x-ndjson",
+        chunks=tuple(json.dumps(frame).encode() + b"\n" for frame in frames),
+        drop_connection=drop_connection,
+    )
+
+
+def _is_generate(request: Request) -> bool:
+    return (request.method, request.target) == ("POST", "/api/generate")
+
+
+def _marker_of(request: Request) -> str:
+    found: Final = _MARKER.search(request.body.decode())
+    assert found is not None, request.body
+    return found.group(1)
+
+
+def _echo(request: Request) -> Reply:
+    if not _is_generate(request):
+        return Reply(body=b"{}")
+    stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
+    return _generate_reply(_marker_of(request), stream)
+
+
+def _assert_each_prompt_is_instructed_once(received: tuple[Request, ...], markers: frozenset[str]) -> None:
+    generates: Final = tuple(request for request in received if _is_generate(request))
+    assert sorted(_marker_of(request) for request in generates) == sorted(markers)
+    for request in generates:
+        body: Final = _JSON_OBJECT.validate_json(request.body)
+        assert body["format"] == "json", sorted(body)
+        prompt: Final = body["prompt"]
+        assert isinstance(prompt, str)
+        assert prompt.count(_INSTRUCTION) == 1, prompt
+        assert set(_MARKER.findall(prompt)) == {_marker_of(request)}, prompt
+
+
+def _response_id(served: _Served) -> str | None:
+    if served.call.endpoint == "responses":
+        return None
+    if not served.call.stream:
+        identity: Final = _JSON_OBJECT.validate_json(served.text)["id"]
+        assert isinstance(identity, str)
+        return identity
+    for line in served.text.splitlines():
+        if not line.startswith("data: ") or line == "data: [DONE]":
+            continue
+        payload: Final = _JSON_OBJECT.validate_json(line.removeprefix("data: "))
+        if served.call.endpoint == "chat":
+            first: Final = payload["id"]
+            assert isinstance(first, str)
+            return first
+        if payload.get("type") == "message_start":
+            message: Final = payload["message"]
+            assert isinstance(message, dict) and isinstance(message["id"], str)
+            return message["id"]
+    raise AssertionError(served.text)
+
+
+def _spend_statuses(model: str, expected: int) -> MappingProxyType[str, str]:
+    rows: Final = eventually(
+        lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
+        lambda found: len(found) >= expected,
+        seconds=70,
+    )
+    statuses: Final = MappingProxyType({str(row["request_id"]): str(row["status"]) for row in rows})
+    assert len(statuses) == len(rows) == expected, rows
+    return statuses
+
+
+def _successes(statuses: MappingProxyType[str, str]) -> frozenset[str]:
+    return frozenset(identity for identity, status in statuses.items() if status == "success")
+
+
+async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
+    async with client.stream(
+        "POST",
+        _path(call.endpoint),
+        json=_body(model, call),
+        headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"},
+    ) as response:
+        raw: Final = await response.aread()
+    return _Served(call=call, status=response.status_code, text=raw.decode())
+
+
+async def _burst(
+    base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False
+) -> tuple[_Served, ...]:
+    async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
+        results: Final = await asyncio.gather(
+            *(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors
+        )
+    for result in results:
+        assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
+    return tuple(result for result in results if isinstance(result, _Served))
+
+
+def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]:
+    return tuple(
+        _Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex)
+        for index in range(count)
+    )
+
+
+def _assert_answered_with_its_own_marker(served: _Served) -> None:
+    assert served.status == 200, served.text
+    assert set(_MARKER.findall(served.text)) == {served.call.marker}, served.text
+
+
+async def test_concurrent_tool_result_turns_across_endpoints_each_get_their_own_instructed_prompt(
+    gateway: Gateway,
+) -> None:
+    calls: Final = _calls(24, ("chat", "messages", "responses"), lambda index: index % 2 == 0)
+    with wire_server(_echo) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
+        assert len(served) == 24
+        for item in served:
+            _assert_answered_with_its_own_marker(item)
+        _assert_each_prompt_is_instructed_once(wire.drain(), frozenset(call.marker for call in calls))
+        known: Final = frozenset(identity for identity in map(_response_id, served) if identity is not None)
+        assert len(known) == 16, known
+        statuses: Final = _spend_statuses(model, 24)
+        assert _successes(statuses) == frozenset(statuses), statuses
+        assert known <= _successes(statuses)
+
+
+async def test_dropped_ollama_connections_fail_their_callers_and_the_rest_keep_their_prompts(gateway: Gateway) -> None:
+    calls: Final = _calls(12, ("chat",), lambda index: index % 2 == 1)
+    dropped: Final = frozenset(call.marker for index, call in enumerate(calls) if index % 3 == 0)
+
+    def respond(request: Request) -> Reply:
+        if not _is_generate(request):
+            return Reply(body=b"{}")
+        marker: Final = _marker_of(request)
+        stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
+        return _generate_reply(marker, stream, drop_connection=marker in dropped)
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
+        assert len(served) == 12
+        for item in served:
+            if item.call.marker in dropped:
+                assert item.status == 500, item.text
+                assert "APIConnectionError" in item.text and "marker-" not in item.text, item.text
+            else:
+                _assert_answered_with_its_own_marker(item)
+        recovery: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
+        (recovered,) = await _burst(str(gateway.client.base_url), gateway.key, model, (recovery,))
+        _assert_answered_with_its_own_marker(recovered)
+        _assert_each_prompt_is_instructed_once(wire.drain(), frozenset(call.marker for call in (*calls, recovery)))
+        answered: Final = tuple(item for item in served if item.call.marker not in dropped)
+        survivors: Final = frozenset(
+            identity for identity in map(_response_id, (*answered, recovered)) if identity is not None
+        )
+        assert len(survivors) == 9, survivors
+        statuses: Final = _spend_statuses(model, 13)
+        assert _successes(statuses) == survivors, statuses
+        assert sum(status == "failure" for status in statuses.values()) == len(dropped), statuses
+
+
+def _chaos_config(wire: Wire, tmp_path: Path) -> Path:
+    base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+    deployment: Final = {
+        "model_name": _CONFIG_MODEL,
+        "litellm_params": {"model": f"ollama/{_BACKEND}", "api_base": wire.url, "api_key": _API_KEY},
+    }
+    path: Final = tmp_path / "ollama-prompt-tools-chaos.yaml"
+    path.write_text(yaml.safe_dump({**base, "model_list": [deployment]}))
+    return path
+
+
+def _open_upstream_connections(pid: int, upstream: str) -> int:
+    port: Final = urlsplit(upstream).port
+    return sum(
+        1
+        for connection in psutil.Process(pid).net_connections(kind="tcp")
+        if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
+    )
+
+
+@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
+async def test_worker_sigkill_mid_burst_leaves_the_sibling_instructing_ollama(gateway: Gateway, tmp_path: Path) -> None:
+    calls: Final = _calls(20, ("chat",), lambda _: False)
+    release: Final = threading.Event()
+    held_markers: Final[SimpleQueue[str]] = SimpleQueue()
+
+    def held(request: Request) -> Reply:
+        if not _is_generate(request):
+            return Reply(body=b"{}")
+        held_markers.put(_marker_of(request))
+        assert release.wait(timeout=60), "The burst was never released"
+        return _echo(request)
+
+    with wire_server(held) as wire:
+        path: Final = _chaos_config(wire, tmp_path)
+        with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned:
+            candidate: Final = owned.gateway
+            workers: Final = eventually(
+                lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
+                lambda pids: len(pids) == 2,
+                seconds=30,
+            )
+            burst: Final = asyncio.create_task(
+                _burst(
+                    str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True
+                )
+            )
+            await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60)
+            held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
+            assert sum(held_by.values()) == 20, held_by
+            victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
+            victim: Final = psutil.Process(victim_pid)
+            victim.suspend()
+            victim.send_signal(signal.SIGKILL)
+            release.set()
+            served: Final = await burst
+            assert held_by[survivor_pid] >= 10, held_by
+            assert len(served) == held_by[survivor_pid], (held_by, len(served))
+            for item in served:
+                _assert_answered_with_its_own_marker(item)
+            follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
+            (answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,))
+            _assert_answered_with_its_own_marker(answered)
+            _assert_each_prompt_is_instructed_once(wire.drain(), frozenset(call.marker for call in (*calls, follow_up)))

tests/integration/providers/test_ollama_prompt_tools_wire.py

@@ -0,0 +1,796 @@
+import itertools
+import json
+import uuid
+from collections.abc import Callable, Iterator, Sequence
+from contextlib import contextmanager
+from typing import Final
+
+import anthropic
+import openai
+import pytest
+from openai.types.chat import ChatCompletionChunk
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
+from integration._support.client import Gateway, eventually
+from integration._support.database import read_rows
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+_BACKEND: Final = "llama3-prompt-tools"
+_API_KEY: Final = "synthetic-ollama-key"
+_INSTRUCTION: Final = (
+    'To call a function, reply with JSON ONLY in this format {"name": "function_name", '
+    '"arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, '
+    "reply to the user in plain text instead of calling a function again. "
+    "The following functions are available to you:"
+)
+_QUESTION: Final = "What is the weather in Paris?"
+_RESULT: Final = "Paris: 22 degrees Celsius, clear skies"
+_ANSWER: Final = "Paris is 22 degrees Celsius with clear skies."
+_CALL_ID: Final = "call_prompt_tools_1"
+_ARGUMENTS: Final[dict[str, JsonValue]] = {"city": "Paris"}
+_CALL_JSON: Final = json.dumps({"name": "get_weather", "arguments": _ARGUMENTS})
+_CALL_JSON_FIELDS: Final = frozenset({"get_weather"})
+_PARAMETERS: Final[dict[str, JsonValue]] = {
+    "type": "object",
+    "properties": {"city": {"type": "string"}},
+    "required": ["city"],
+}
+_WEATHER_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "function": {"name": "get_weather", "description": "Weather for a city", "parameters": _PARAMETERS},
+}
+_TIME_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "function": {"name": "get_time", "description": "Local time for a city", "parameters": _PARAMETERS},
+}
+_ANTHROPIC_TOOL: Final[dict[str, JsonValue]] = {
+    "name": "get_weather",
+    "description": "Weather for a city",
+    "input_schema": _PARAMETERS,
+}
+_RESPONSES_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "name": "get_weather",
+    "description": "Weather for a city",
+    "parameters": _PARAMETERS,
+}
+_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True}
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_RAW_ANTHROPIC_EVENTS: Final = frozenset(
+    {
+        "message_start",
+        "content_block_start",
+        "content_block_delta",
+        "content_block_stop",
+        "message_delta",
+        "message_stop",
+    }
+)
+
+
+def _generate_reply(text: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "model": _BACKEND,
+                "created_at": "2026-10-07T00:00:00Z",
+                "response": text,
+                "done": True,
+                "done_reason": "stop",
+                "prompt_eval_count": 30,
+                "eval_count": 12,
+            }
+        ).encode()
+    )
+
+
+def _streamed_reply(text: str) -> Reply:
+    pieces: Final = tuple(text[index : index + 7] for index in range(0, len(text), 7))
+    lines: Final = tuple(
+        json.dumps({"model": _BACKEND, "created_at": "2026-10-07T00:00:00Z", "response": piece, "done": False}).encode()
+        + b"\n"
+        for piece in pieces
+    )
+    final: Final = (
+        json.dumps(
+            {
+                "model": _BACKEND,
+                "created_at": "2026-10-07T00:00:00Z",
+                "response": "",
+                "done": True,
+                "done_reason": "stop",
+                "prompt_eval_count": 30,
+                "eval_count": 12,
+            }
+        ).encode()
+        + b"\n"
+    )
+    return Reply(content_type="application/x-ndjson", chunks=(*lines, final))
+
+
+def _is_generate(request: Request) -> bool:
+    return (request.method, request.target) == ("POST", "/api/generate")
+
+
+@contextmanager
+def _ollama_server(respond: Callable[[Request], Reply]) -> Iterator[Wire]:
+    with wire_server(lambda request: respond(request) if _is_generate(request) else Reply(body=b"{}")) as wire:
+        yield wire
+
+
+def _generate_calls(wire: Wire) -> tuple[Request, ...]:
+    return tuple(request for request in wire.drain() if _is_generate(request))
+
+
+def _only_generate(wire: Wire) -> dict[str, JsonValue]:
+    received: Final = _generate_calls(wire)
+    assert len(received) == 1, [(request.method, request.target) for request in received]
+    assert received[0].headers["authorization"] == f"Bearer {_API_KEY}"
+    return _JSON_OBJECT.validate_json(received[0].body)
+
+
+def _prompt_of(body: dict[str, JsonValue]) -> str:
+    assert body["model"] == _BACKEND
+    assert body["format"] == "json"
+    assert "tools" not in body and "messages" not in body, sorted(body)
+    prompt: Final = body["prompt"]
+    assert isinstance(prompt, str)
+    return prompt
+
+
+def _assert_instructed_once(prompt: str, *tool_names: str) -> None:
+    assert prompt.count(_INSTRUCTION) == 1, prompt
+    assert prompt.count("### System:") == 1, prompt
+    for name in tool_names:
+        assert f"'name': '{name}'" in prompt, prompt
+
+
+def _assert_tool_turn(prompt: str, result: str = _RESULT) -> None:
+    _assert_instructed_once(prompt, "get_weather")
+    assert f"### User:\n{_QUESTION}\n\n### Assistant:\n{_CALL_JSON}\n\n### User:\n{result}\n\n" in prompt, prompt
+
+
+def _spend_row(identity: str) -> dict[str, JsonValue]:
+    rows: Final = eventually(
+        lambda: read_rows(
+            'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
+            (identity,),
+        ),
+        lambda found: len(found) == 1,
+        seconds=70,
+    )
+    return rows[0]
+
+
+def _model_spend_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
+    rows: Final = eventually(
+        lambda: read_rows(
+            'SELECT request_id, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
+            (model,),
+        ),
+        lambda found: len(found) >= expected,
+        seconds=70,
+    )
+    assert len({row["request_id"] for row in rows}) == len(rows) == expected, rows
+    return rows
+
+
+def _billed(model: str) -> dict[str, JsonValue]:
+    return {"model_group": model, "status": "success", "prompt_tokens": 30, "completion_tokens": 12}
+
+
+def _openai_client(gateway: Gateway) -> openai.OpenAI:
+    return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
+
+
+def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI:
+    return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
+
+
+def _anthropic_client(gateway: Gateway) -> anthropic.Anthropic:
+    return anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+
+
+def _async_anthropic_client(gateway: Gateway) -> anthropic.AsyncAnthropic:
+    return anthropic.AsyncAnthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+
+
+def _stream_choices(chunks: Sequence[ChatCompletionChunk]) -> Iterator[ChunkChoice]:
+    for chunk in chunks:
+        yield from chunk.choices
+
+
+def _delta_tool_calls(choices: Sequence[ChunkChoice]) -> Iterator[ChoiceDeltaToolCall]:
+    for choice in choices:
+        yield from choice.delta.tool_calls or ()
+
+
+def _first_turn() -> list[dict[str, JsonValue]]:
+    return [{"role": "user", "content": _QUESTION}]
+
+
+def _second_turn(result: JsonValue = _RESULT) -> list[dict[str, JsonValue]]:
+    return [
+        {"role": "user", "content": _QUESTION},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {
+                    "id": _CALL_ID,
+                    "type": "function",
+                    "function": {"name": "get_weather", "arguments": json.dumps(_ARGUMENTS)},
+                }
+            ],
+        },
+        {"role": "tool", "tool_call_id": _CALL_ID, "content": result},
+    ]
+
+
+def _anthropic_second_turn() -> list[dict[str, JsonValue]]:
+    return [
+        {"role": "user", "content": _QUESTION},
+        {
+            "role": "assistant",
+            "content": [{"type": "tool_use", "id": _CALL_ID, "name": "get_weather", "input": _ARGUMENTS}],
+        },
+        {"role": "user", "content": [{"type": "tool_result", "tool_use_id": _CALL_ID, "content": _RESULT}]},
+    ]
+
+
+def _responses_second_turn() -> list[dict[str, JsonValue]]:
+    return [
+        {"role": "user", "content": _QUESTION},
+        {"type": "function_call", "call_id": _CALL_ID, "name": "get_weather", "arguments": json.dumps(_ARGUMENTS)},
+        {"type": "function_call_output", "call_id": _CALL_ID, "output": _RESULT},
+    ]
+
+
+def _post(gateway: Gateway, path: str, body: dict[str, JsonValue], key: str | None = None) -> tuple[int, str]:
+    response: Final = gateway.request("POST", path, {**body, "cache": _NO_CACHE}, key=key)
+    return response.status_code, response.text
+
+
+def _post_chat(
+    gateway: Gateway, model: str, messages: Sequence[dict[str, JsonValue]], **extra: JsonValue
+) -> dict[str, JsonValue]:
+    code, text = _post(gateway, "/v1/chat/completions", {"model": model, "messages": list(messages), **extra})
+    assert code == 200, text
+    return _JSON_OBJECT.validate_json(text)
+
+
+def test_openai_sdk_tool_request_reaches_ollama_as_an_instructed_prompt(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        completion: Final = _openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_first_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        choice: Final = completion.choices[0]
+        assert choice.finish_reason == "tool_calls"
+        assert choice.message.tool_calls is not None and len(choice.message.tool_calls) == 1
+        call: Final = choice.message.tool_calls[0]
+        assert call.type == "function"
+        assert call.function.name == "get_weather"
+        assert json.loads(call.function.arguments) == _ARGUMENTS
+        assert completion.usage is not None
+        assert (completion.usage.prompt_tokens, completion.usage.completion_tokens) == (30, 12)
+        body: Final = _only_generate(wire)
+        assert body["stream"] is False
+        prompt: Final = _prompt_of(body)
+        _assert_instructed_once(prompt, "get_weather")
+        assert f"### User:\n{_QUESTION}\n\n" in prompt, prompt
+        assert "Weather for a city" in prompt, prompt
+        assert _spend_row(completion.id) == _billed(model)
+
+
+def test_openai_sdk_tool_result_turn_gets_a_plain_text_answer(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        completion: Final = _openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        choice: Final = completion.choices[0]
+        assert choice.finish_reason == "stop"
+        assert choice.message.content == _ANSWER
+        assert choice.message.tool_calls is None
+        _assert_tool_turn(_prompt_of(_only_generate(wire)))
+        assert _spend_row(completion.id) == _billed(model)
+
+
+async def test_async_openai_sdk_stream_flushes_the_held_tool_call_once(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _streamed_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        stream: Final = await _async_openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_first_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            stream=True,
+            stream_options={"include_usage": True},
+            extra_body={"cache": _NO_CACHE},
+        )
+        chunks: Final = [chunk async for chunk in stream]
+        assert {chunk.id for chunk in chunks} == {chunks[0].id}
+        choices: Final = tuple(_stream_choices(chunks))
+        deltas: Final = tuple(_delta_tool_calls(choices))
+        assert len(deltas) == 1, deltas
+        assert deltas[0].function is not None and deltas[0].function.name == "get_weather"
+        assert json.loads(deltas[0].function.arguments or "") == _ARGUMENTS
+        assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["tool_calls"]
+        usages: Final = [chunk.usage for chunk in chunks if chunk.usage is not None]
+        assert [(usage.prompt_tokens, usage.completion_tokens) for usage in usages] == [(30, 12)]
+        body: Final = _only_generate(wire)
+        assert body["stream"] is True
+        _assert_instructed_once(_prompt_of(body), "get_weather")
+        assert _spend_row(chunks[0].id) == _billed(model)
+
+
+async def test_async_openai_sdk_stream_answers_the_tool_result_in_plain_text(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        stream: Final = await _async_openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            stream=True,
+            stream_options={"include_usage": True},
+            extra_body={"cache": _NO_CACHE},
+        )
+        chunks: Final = [chunk async for chunk in stream]
+        choices: Final = tuple(_stream_choices(chunks))
+        assert "".join(choice.delta.content or "" for choice in choices) == _ANSWER
+        assert tuple(_delta_tool_calls(choices)) == ()
+        assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"]
+        body: Final = _only_generate(wire)
+        assert body["stream"] is True
+        _assert_tool_turn(_prompt_of(body))
+        assert _spend_row(chunks[0].id) == _billed(model)
+
+
+def test_anthropic_sdk_tool_request_comes_back_as_tool_use(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        message: Final = _anthropic_client(gateway).messages.create(
+            model=model,
+            max_tokens=64,
+            messages=_first_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_ANTHROPIC_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert message.stop_reason == "tool_use"
+        assert [block.type for block in message.content] == ["tool_use"]
+        block: Final = message.content[0]
+        assert block.type == "tool_use"
+        assert block.name == "get_weather"
+        assert block.input == _ARGUMENTS
+        assert (message.usage.input_tokens, message.usage.output_tokens) == (30, 12)
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        assert f"### User:\n{_QUESTION}\n\n" in prompt, prompt
+        assert _spend_row(message.id) == _billed(model)
+
+
+def test_anthropic_sdk_tool_result_turn_ends_with_text(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        message: Final = _anthropic_client(gateway).messages.create(
+            model=model,
+            max_tokens=64,
+            messages=_anthropic_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_ANTHROPIC_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert message.stop_reason == "end_turn"
+        assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _ANSWER)]
+        _assert_tool_turn(_prompt_of(_only_generate(wire)))
+        assert _spend_row(message.id) == _billed(model)
+
+
+async def test_async_anthropic_sdk_stream_emits_the_tool_use_block(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _streamed_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        async with _async_anthropic_client(gateway).messages.stream(
+            model=model,
+            max_tokens=64,
+            messages=_first_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_ANTHROPIC_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        ) as stream:
+            events: Final = [event async for event in stream if event.type in _RAW_ANTHROPIC_EVENTS]
+            final: Final = await stream.get_final_message()
+        starts: Final = [event for event in events if event.type == "content_block_start"]
+        assert [event.content_block.type for event in starts] == ["tool_use"], [event.type for event in events]
+        assert any(
+            event.type == "content_block_delta" and event.delta.type == "input_json_delta" for event in events
+        ), [event.type for event in events]
+        assert final.stop_reason == "tool_use"
+        block: Final = final.content[0]
+        assert block.type == "tool_use" and block.name == "get_weather" and block.input == _ARGUMENTS
+        body: Final = _only_generate(wire)
+        assert body["stream"] is True
+        _assert_instructed_once(_prompt_of(body), "get_weather")
+        assert _spend_row(final.id) == _billed(model)
+
+
+async def test_async_anthropic_sdk_stream_answers_the_tool_result_in_text(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        async with _async_anthropic_client(gateway).messages.stream(
+            model=model,
+            max_tokens=64,
+            messages=_anthropic_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_ANTHROPIC_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        ) as stream:
+            texts: Final = [event.text async for event in stream if event.type == "text"]
+            final: Final = await stream.get_final_message()
+        assert "".join(texts) == _ANSWER
+        assert final.stop_reason == "end_turn"
+        assert [(block.type, getattr(block, "text", None)) for block in final.content] == [("text", _ANSWER)]
+        body: Final = _only_generate(wire)
+        assert body["stream"] is True
+        _assert_tool_turn(_prompt_of(body))
+        assert _spend_row(final.id) == _billed(model)
+
+
+def test_openai_sdk_responses_tool_request_comes_back_as_a_function_call(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        response: Final = _openai_client(gateway).responses.create(
+            model=model,
+            input=_QUESTION,
+            tools=[_RESPONSES_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            store=False,
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert [item.type for item in response.output] == ["function_call"]
+        item: Final = response.output[0]
+        assert item.type == "function_call"
+        assert item.name == "get_weather"
+        assert json.loads(item.arguments) == _ARGUMENTS
+        assert response.usage is not None and (response.usage.input_tokens, response.usage.output_tokens) == (30, 12)
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        assert f"### User:\n{_QUESTION}\n\n" in prompt, prompt
+        assert _model_spend_rows(model, 1)[0]["status"] == "success"
+
+
+def test_openai_sdk_responses_function_output_turn_gets_a_message(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        response: Final = _openai_client(gateway).responses.create(
+            model=model,
+            input=_responses_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON input items
+            tools=[_RESPONSES_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            store=False,
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert [item.type for item in response.output] == ["message"]
+        assert response.output_text == _ANSWER
+        _assert_tool_turn(_prompt_of(_only_generate(wire)))
+        rows: Final = _model_spend_rows(model, 1)
+        assert (rows[0]["status"], rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == ("success", 30, 12)
+
+
+async def test_async_openai_sdk_responses_stream_emits_the_function_call_item(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _streamed_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        stream: Final = await _async_openai_client(gateway).responses.create(
+            model=model,
+            input=_QUESTION,
+            tools=[_RESPONSES_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            store=False,
+            stream=True,
+            extra_body={"cache": _NO_CACHE},
+        )
+        events: Final = [event async for event in stream]
+        done_items: Final = [event.item for event in events if event.type == "response.output_item.done"]
+        assert [item.type for item in done_items] == ["function_call"], [event.type for event in events]
+        item: Final = done_items[0]
+        assert item.type == "function_call" and item.name == "get_weather"
+        assert json.loads(item.arguments) == _ARGUMENTS
+        completed: Final = [event for event in events if event.type == "response.completed"]
+        assert len(completed) == 1
+        final_calls: Final = [item for item in completed[0].response.output if item.type == "function_call"]
+        assert [(item.name, json.loads(item.arguments)) for item in final_calls] == [("get_weather", _ARGUMENTS)]
+        assert completed[0].response.output_text == ""
+        body: Final = _only_generate(wire)
+        assert body["stream"] is True
+        _assert_instructed_once(_prompt_of(body), "get_weather")
+        assert _model_spend_rows(model, 1)[0]["status"] == "success"
+
+
+async def test_async_openai_sdk_responses_stream_answers_the_function_output_in_text(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        stream: Final = await _async_openai_client(gateway).responses.create(
+            model=model,
+            input=_responses_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON input items
+            tools=[_RESPONSES_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            store=False,
+            stream=True,
+            extra_body={"cache": _NO_CACHE},
+        )
+        events: Final = [event async for event in stream]
+        assert "".join(event.delta for event in events if event.type == "response.output_text.delta") == _ANSWER
+        completed: Final = [event for event in events if event.type == "response.completed"]
+        assert len(completed) == 1 and completed[0].response.output_text == _ANSWER
+        done_types: Final = [event.type for event in events if event.type == "response.output_item.done"]
+        assert done_types == ["response.output_item.done"]
+        body: Final = _only_generate(wire)
+        assert body["stream"] is True
+        _assert_tool_turn(_prompt_of(body))
+        assert _model_spend_rows(model, 1)[0]["status"] == "success"
+
+
+def test_legacy_functions_param_is_instructed_the_same_way(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        function: Final = _WEATHER_TOOL["function"]
+        payload: Final = _post_chat(gateway, model, _first_turn(), functions=[function])
+        choices: Final = payload["choices"]
+        assert isinstance(choices, list) and len(choices) == 1
+        assert _CALL_JSON_FIELDS <= set(json.dumps(choices[0]).split('"')), choices
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        assert "Weather for a city" in prompt, prompt
+        identity: Final = payload["id"]
+        assert isinstance(identity, str)
+        assert _spend_row(identity) == _billed(model)
+
+
+def test_string_system_message_keeps_one_system_section_with_the_instruction(gateway: Gateway) -> None:
+    system: Final = f"You are a terse weather bot {uuid.uuid4().hex}."
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        _post_chat(gateway, model, [{"role": "system", "content": system}, *_first_turn()], tools=[_WEATHER_TOOL])
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        assert f"### System:\n{system} {_INSTRUCTION}\n" in prompt, prompt
+
+
+def test_list_system_message_keeps_one_system_section_with_the_instruction(gateway: Gateway) -> None:
+    system: Final = f"You are a terse weather bot {uuid.uuid4().hex}."
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        _post_chat(
+            gateway,
+            model,
+            [{"role": "system", "content": [{"type": "text", "text": system}]}, *_first_turn()],
+            tools=[_WEATHER_TOOL],
+        )
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        section: Final = prompt.split("### System:\n", 1)[1]
+        assert section.startswith(system), section
+        assert section.count(_INSTRUCTION) == 1, section
+
+
+def test_two_tools_are_both_listed_under_one_instruction(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        _post_chat(gateway, model, _first_turn(), tools=[_WEATHER_TOOL, _TIME_TOOL])
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather", "get_time")
+        assert prompt.index("'name': 'get_weather'") < prompt.index("'name': 'get_time'"), prompt
+
+
+def test_unauthenticated_tool_request_never_reaches_ollama(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        code, text = _post(
+            gateway,
+            "/v1/chat/completions",
+            {"model": model, "messages": _first_turn(), "tools": [_WEATHER_TOOL]},
+            key=f"sk-not-a-key-{uuid.uuid4().hex}",
+        )
+        assert code == 401, text
+        assert _generate_calls(wire) == ()
+
+
+def test_ollama_model_not_found_reaches_the_caller_after_one_attempt(gateway: Gateway) -> None:
+    message: Final = f"model '{_BACKEND}' not found {uuid.uuid4().hex}"
+    reply: Final = Reply(status=404, body=json.dumps({"error": message}).encode())
+    with _ollama_server(lambda _: reply) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        code, text = _post(
+            gateway, "/v1/chat/completions", {"model": model, "messages": _first_turn(), "tools": [_WEATHER_TOOL]}
+        )
+        assert code == 404, text
+        assert message in text, text
+        _assert_instructed_once(_prompt_of(_only_generate(wire)), "get_weather")
+
+
+def test_ollama_server_error_on_the_tool_result_turn_does_not_take_the_deployment_down(gateway: Gateway) -> None:
+    attempts: Final = itertools.count()
+    failure: Final = f"internal failure {uuid.uuid4().hex}"
+
+    def respond(_: Request) -> Reply:
+        if next(attempts) == 0:
+            return Reply(status=500, body=json.dumps({"error": failure}).encode())
+        return _generate_reply(_ANSWER)
+
+    with _ollama_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        code, text = _post(
+            gateway, "/v1/chat/completions", {"model": model, "messages": _second_turn(), "tools": [_WEATHER_TOOL]}
+        )
+        assert code == 500, text
+        assert failure in text, text
+        payload: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL])
+        choices: Final = payload["choices"]
+        assert isinstance(choices, list) and len(choices) == 1
+        assert json.dumps(choices[0]).count(_ANSWER) == 1, choices
+        prompts: Final = [_prompt_of(_JSON_OBJECT.validate_json(request.body)) for request in _generate_calls(wire)]
+        assert len(prompts) == 2, prompts
+        for prompt in prompts:
+            _assert_tool_turn(prompt)
+
+
+@pytest.mark.parametrize(
+    ("result", "forwarded"),
+    [
+        pytest.param("", None, id="empty-string-drops-the-section"),
+        pytest.param("r" * 5120, "r" * 5120, id="5kb-string-forwarded-intact"),
+        pytest.param(
+            [{"type": "text", "text": "Paris: 22 degrees"}, {"type": "text", "text": "clear skies"}],
+            "Paris: 22 degreesclear skies",
+            id="text-parts-joined",
+        ),
+    ],
+)
+def test_tool_result_content_shapes_reach_the_prompt(
+    gateway: Gateway, result: JsonValue, forwarded: str | None
+) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        payload: Final = _post_chat(gateway, model, _second_turn(result), tools=[_WEATHER_TOOL])
+        assert json.dumps(payload["choices"]).count(_ANSWER) == 1, payload
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        if forwarded is None:
+            assert f"### User:\n{_QUESTION}\n\n### Assistant:\n{_CALL_JSON}\n\n### System:\n" in prompt, prompt
+        else:
+            _assert_tool_turn(prompt, forwarded)
+
+
+def test_the_same_tool_result_twice_is_forwarded_twice_under_one_instruction(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        messages: Final = [*_second_turn(), {"role": "tool", "tool_call_id": _CALL_ID, "content": _RESULT}]
+        _post_chat(gateway, model, messages, tools=[_WEATHER_TOOL])
+        prompt: Final = _prompt_of(_only_generate(wire))
+        _assert_instructed_once(prompt, "get_weather")
+        assert prompt.count(_RESULT) == 2, prompt
+        assert prompt.count("### User:") == 2 and prompt.count("### Assistant:") == 1, prompt
+
+
+def test_a_second_function_call_after_the_result_is_surfaced_as_tool_calls(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        completion: Final = _openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        choice: Final = completion.choices[0]
+        assert choice.finish_reason == "tool_calls"
+        assert choice.message.tool_calls is not None and [call.function.name for call in choice.message.tool_calls] == [
+            "get_weather"
+        ]
+        _assert_tool_turn(_prompt_of(_only_generate(wire)))
+        assert _spend_row(completion.id) == _billed(model)
+
+
+def test_a_non_function_json_answer_is_returned_as_text(gateway: Gateway) -> None:
+    answer: Final = json.dumps({"city": "Paris", "temperature_c": 22, "sky": "clear"})
+    with _ollama_server(lambda _: _generate_reply(answer)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        completion: Final = _openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        choice: Final = completion.choices[0]
+        assert choice.finish_reason == "stop"
+        assert choice.message.tool_calls is None
+        assert choice.message.content is not None and json.loads(choice.message.content) == json.loads(answer)
+        _assert_tool_turn(_prompt_of(_only_generate(wire)))
+        assert _spend_row(completion.id) == _billed(model)
+
+
+def test_int_tool_result_content_fails_in_the_response_body_and_leaves_the_deployment_serving(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        code, text = _post(
+            gateway, "/v1/chat/completions", {"model": model, "messages": _second_turn(22), "tools": [_WEATHER_TOOL]}
+        )
+        assert code >= 400, text
+        error: Final = _JSON_OBJECT.validate_json(text)["error"]
+        assert isinstance(error, dict) and isinstance(error["message"], str) and error["message"], text
+        assert _generate_calls(wire) == ()
+        payload: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL])
+        assert json.dumps(payload["choices"]).count(_ANSWER) == 1, payload
+        _assert_tool_turn(_prompt_of(_only_generate(wire)))
+
+
+def test_ollama_chat_keeps_native_tools_and_gets_no_instruction(gateway: Gateway) -> None:
+    reply: Final = Reply(
+        body=json.dumps(
+            {
+                "model": _BACKEND,
+                "created_at": "2026-10-07T00:00:00Z",
+                "message": {"role": "assistant", "content": _ANSWER},
+                "done": True,
+                "prompt_eval_count": 30,
+                "eval_count": 12,
+            }
+        ).encode()
+    )
+    with wire_server(lambda _: reply) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama_chat/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        completion: Final = _openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_second_turn(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            tools=[_WEATHER_TOOL],  # pyright: ignore[reportArgumentType]  # plain JSON tool
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert completion.choices[0].message.content == _ANSWER
+        received: Final = tuple(request for request in wire.drain() if request.method == "POST")
+        assert [request.target for request in received] == ["/api/chat"]
+        body: Final = _JSON_OBJECT.validate_json(received[0].body)
+        assert "prompt" not in body and "format" not in body, sorted(body)
+        tools: Final = body["tools"]
+        assert isinstance(tools, list) and len(tools) == 1
+        messages: Final = body["messages"]
+        assert isinstance(messages, list) and [item["role"] for item in messages if isinstance(item, dict)] == [
+            "user",
+            "assistant",
+            "tool",
+        ]
+        assert "function_name" not in received[0].body.decode(), received[0].body
+        assert _spend_row(completion.id) == _billed(model)
+
+
+def test_identical_uncached_tool_result_turns_are_each_forwarded_and_billed_once(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+        first: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL])
+        second: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL])
+        assert first["id"] != second["id"], (first["id"], second["id"])
+        prompts: Final = [_prompt_of(_JSON_OBJECT.validate_json(request.body)) for request in _generate_calls(wire)]
+        assert len(prompts) == 2, prompts
+        for prompt in prompts:
+            _assert_tool_turn(prompt)
+        for payload in (first, second):
+            identity: Final = payload["id"]
+            assert isinstance(identity, str)
+            assert _spend_row(identity) == _billed(model)
+
+
+def test_the_cell_deployment_is_gone_after_its_scenario(gateway: Gateway) -> None:
+    with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire:
+        with gateway.scenario() as scenario:
+            model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
+            _post_chat(gateway, model, _first_turn(), tools=[_WEATHER_TOOL])
+            _only_generate(wire)
+        listed: Final = eventually(
+            lambda: [entry["model_name"] for entry in _deployments(gateway) if isinstance(entry, dict)],
+            lambda names: model not in names,
+            seconds=70,
+        )
+        assert model not in listed
+
+
+def _deployments(gateway: Gateway) -> list[JsonValue]:
+    data: Final = gateway.get("/model/info")["data"]
+    assert isinstance(data, list)
+    return data

tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -51,6 +51,17 @@ def test_function_call_prompt_preserves_append_failure_for_non_string_content()
         function_call_prompt(messages, [])
 
 
+def test_function_call_prompt_lets_the_model_answer_after_a_function_result() -> None:
+    messages: Final[list[dict[str, object]]] = [{"role": "system", "content": "Be terse."}]
+
+    prompted: Final = function_call_prompt(messages, [{"name": "get_weather"}])
+
+    system: Final = str(prompted[0]["content"])
+    assert "JSON OUTPUT ONLY" not in system
+    assert "reply to the user in plain text instead of calling a function again" in system
+    assert "{'name': 'get_weather'}" in system
+
+
 @pytest.mark.parametrize(
     ("thought_signature", "expected"),
     [

tests/unit/llms/base_llm/chat/test_attribution_headers.py

@@ -0,0 +1,210 @@
+"""
+Provider attribution headers (`BaseConfig.get_attribution_headers`) must reach
+the outbound request on every OpenAI-compatible chat path, and a caller header
+with the same name must win.
+
+Requests go through a real `litellm.completion` into an in-process httpx
+transport that records what would have been sent, because the default path
+(OpenAI SDK) never calls `validate_environment`.
+"""
+
+import json
+from collections.abc import AsyncIterable, Iterable
+from typing import Final, cast
+
+import httpx
+import openai
+import pytest
+
+import litellm
+from litellm.llms.base_llm.chat.transformation import with_attribution_headers
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+
+_API_BASE: Final = "https://provider.invalid/v1"
+_COMPLETION_BODY: Final = json.dumps(
+    {
+        "id": "chatcmpl-1",
+        "object": "chat.completion",
+        "created": 1,
+        "model": "m",
+        "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+        "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+    }
+).encode()
+_STREAM_BODY: Final = (
+    "data: "
+    + json.dumps(
+        {
+            "id": "chatcmpl-1",
+            "object": "chat.completion.chunk",
+            "created": 1,
+            "model": "m",
+            "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+        }
+    )
+    + "\n\ndata: [DONE]\n\n"
+).encode()
+
+
+class _HeaderCapturingTransport(httpx.BaseTransport, httpx.AsyncBaseTransport):
+    """Records each outbound request's headers and answers like a chat completions server."""
+
+    def __init__(self) -> None:
+        self.sent: tuple[httpx.Headers, ...] = ()
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        return self._respond(request, request.read())
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        return self._respond(request, await request.aread())
+
+    def _respond(self, request: httpx.Request, body: bytes) -> httpx.Response:
+        self.sent = (*self.sent, request.headers)
+        if json.loads(body).get("stream"):
+            return httpx.Response(200, content=_STREAM_BODY, headers={"content-type": "text/event-stream"})
+        return httpx.Response(200, content=_COMPLETION_BODY, headers={"content-type": "application/json"})
+
+    def last(self, header: str) -> list[str]:
+        return self.sent[-1].get_list(header)
+
+
+def _client(transport: _HeaderCapturingTransport, path: str, is_async: bool) -> object:
+    if path == "sdk":
+        if is_async:
+            return openai.AsyncOpenAI(
+                api_key="k", base_url=_API_BASE, http_client=httpx.AsyncClient(transport=transport)
+            )
+        return openai.OpenAI(api_key="k", base_url=_API_BASE, http_client=httpx.Client(transport=transport))
+    if is_async:
+        return AsyncHTTPHandler(transport=transport)
+    return HTTPHandler(client=httpx.Client(transport=transport))
+
+
+@pytest.fixture
+def transport() -> _HeaderCapturingTransport:
+    return _HeaderCapturingTransport()
+
+
+@pytest.fixture(params=["sdk", "http_handler"])
+def handler_path(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> str:
+    if request.param == "http_handler":
+        monkeypatch.setenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", "true")
+    else:
+        monkeypatch.delenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", raising=False)
+    return request.param
+
+
+_NOVITA_MODEL: Final = "novita/meta-llama/llama-3.3-70b-instruct"
+
+_ATTRIBUTED: Final = [
+    pytest.param(_NOVITA_MODEL, "x-novita-source", id="novita"),
+    pytest.param("perplexity/sonar", "x-pplx-integration", id="perplexity"),
+]
+
+
+def _drain(response: object) -> None:
+    for _ in cast(Iterable[object], response):
+        pass
+
+
+async def _adrain(response: object) -> None:
+    async for _ in cast(AsyncIterable[object], response):
+        pass
+
+
+@pytest.mark.parametrize(("model", "header"), _ATTRIBUTED)
+@pytest.mark.parametrize("stream", [False, True])
+def test_attribution_header_sent_sync(
+    transport: _HeaderCapturingTransport, handler_path: str, model: str, header: str, stream: bool
+) -> None:
+    response: Final = litellm.completion(
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=_API_BASE,
+        api_key="k",
+        stream=stream,
+        client=_client(transport, handler_path, is_async=False),
+    )
+    if stream:
+        _drain(response)
+
+    assert transport.last(header) == ["litellm"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("model", "header"), _ATTRIBUTED)
+@pytest.mark.parametrize("stream", [False, True])
+async def test_attribution_header_sent_async(
+    transport: _HeaderCapturingTransport, handler_path: str, model: str, header: str, stream: bool
+) -> None:
+    response: Final = await litellm.acompletion(
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=_API_BASE,
+        api_key="k",
+        stream=stream,
+        client=_client(transport, handler_path, is_async=True),
+    )
+    if stream:
+        await _adrain(response)
+
+    assert transport.last(header) == ["litellm"]
+
+
+@pytest.mark.parametrize(("model", "header"), _ATTRIBUTED)
+@pytest.mark.parametrize("header_kwarg", ["headers", "extra_headers"])
+def test_caller_header_overrides_attribution_any_casing(
+    transport: _HeaderCapturingTransport, handler_path: str, model: str, header: str, header_kwarg: str
+) -> None:
+    caller_headers: Final = {header.upper(): "my-app"}
+
+    litellm.completion(
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=_API_BASE,
+        api_key="k",
+        client=_client(transport, handler_path, is_async=False),
+        **{header_kwarg: caller_headers},
+    )
+
+    assert transport.last(header) == ["my-app"]
+    assert caller_headers == {header.upper(): "my-app"}
+
+
+def test_provider_without_attribution_sends_none(transport: _HeaderCapturingTransport, handler_path: str) -> None:
+    litellm.completion(
+        model="deepinfra/meta-llama/Meta-Llama-3-8B-Instruct",
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=_API_BASE,
+        api_key="k",
+        client=_client(transport, handler_path, is_async=False),
+    )
+
+    assert transport.last("x-novita-source") == []
+    assert transport.last("x-pplx-integration") == []
+
+
+def test_global_litellm_headers_still_apply_and_are_not_mutated(
+    transport: _HeaderCapturingTransport, handler_path: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    global_headers: Final = {"X-Global": "1"}
+    monkeypatch.setattr(litellm, "headers", global_headers)
+
+    litellm.completion(
+        model=_NOVITA_MODEL,
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=_API_BASE,
+        api_key="k",
+        client=_client(transport, handler_path, is_async=False),
+    )
+
+    assert transport.last("x-global") == ["1"]
+    assert transport.last("x-novita-source") == ["litellm"]
+    assert global_headers == {"X-Global": "1"}
+
+
+def test_with_attribution_headers_returns_headers_unchanged_when_nothing_to_add() -> None:
+    headers: Final = {"A": "1"}
+
+    assert with_attribution_headers({}, headers) is headers
+    assert with_attribution_headers({}, None) is None

tests/unit/proxy/lens/test_endpoints.py

@@ -1,4 +1,5 @@
-from collections.abc import Callable
+import asyncio
+from collections.abc import Callable, Mapping
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
 from typing import Final
@@ -6,21 +7,25 @@ from typing import Final
 import httpx
 import pytest
 from fastapi import HTTPException
-from pydantic import ValidationError
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import Router
 from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.lens.endpoints import (
     claim_due,
+    get_signals,
     list_agents,
+    put_signals,
     read_reviews,
     result,
     run_settings,
     run_window,
     trace_findings,
+    trace_signal_statuses,
     user_scope,
     validate_model,
+    validate_signal_model,
     watchable,
     watching,
     worker_supports_model,
@@ -45,6 +50,7 @@ from litellm.proxy.lens.models import (
     Worker,
 )
 from litellm.proxy.lens.repository import DueLens, Row
+from litellm.proxy.lens.signals import SignalConfig, StoredTraceSignal
 from litellm.proxy.lens.state import claim_job, queue_job, replace_job
 from litellm.rust_bridge.trace.generated.models import ExecutionRow, LensSampleParams
 from litellm.rust_bridge.trace.storage import ClickHouseStorage
@@ -80,6 +86,46 @@ class ResultDatabase:
         return len(self.completed)
 
 
+class SignalStatusDatabase:
+    def __init__(self, config: SignalConfig, rows: Mapping[str, StoredTraceSignal]) -> None:
+        self.config: Final = config
+        self.rows: Final = rows
+        self.saved: Final[asyncio.Queue[tuple[object, ...]]] = asyncio.Queue()
+
+    async def query_raw(self, query: str, *args: object) -> object:
+        if '"LiteLLM_LensSignalConfig"' in query:
+            return ({"data": self.config.model_dump(mode="json")},)
+        payload: Final = args[0]
+        assert isinstance(payload, str)
+        requested: Final = TypeAdapter(tuple[TraceIdentity, ...]).validate_json(payload)
+        return tuple(
+            {"data": row.model_dump(mode="json")}
+            for identity in requested
+            if (row := self.rows.get(identity.trace_id)) is not None
+        )
+
+    async def execute_raw(self, query: str, *args: object) -> int:
+        await self.saved.put(args)
+        return 1
+
+
+def signal_router() -> Router:
+    return Router(
+        model_list=[
+            {
+                "model_name": "decision",
+                "litellm_params": {"model": "openai/test-decision", "api_key": "test-key"},
+                "model_info": {"mode": "evaluation"},
+            },
+            {
+                "model_name": "chat",
+                "litellm_params": {"model": "openai/test-chat", "api_key": "test-key"},
+                "model_info": {"mode": "chat"},
+            },
+        ]
+    )
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("change", ("cancelled", "expired", "reclaimed", "reassigned"))
 async def test_result_cannot_commit_after_losing_ownership_during_evidence_validation(
@@ -526,6 +572,144 @@ async def test_trace_finding_counts_require_investigation_read_access() -> None:
     assert error.value.status_code == 403
 
 
+@pytest.mark.asyncio
+async def test_signal_endpoints_return_statuses_in_request_order_for_admin_viewers(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.proxy import proxy_server
+
+    config: Final = SignalConfig(model="decision")
+    rows: Final = {
+        "pending": StoredTraceSignal(
+            trace_id="pending",
+            config_key=config.key(),
+            span_count=1,
+            claimed_until=NOW + timedelta(minutes=1),
+            data={"status": "pending", "scores": {}, "model": "decision", "error": ""},
+        ),
+        "classified": StoredTraceSignal(
+            trace_id="classified",
+            config_key=config.key(),
+            span_count=1,
+            classified_at=NOW,
+            data={
+                "status": "classified",
+                "scores": {"user_frustration": 0.7, "missing_capability": 0.8},
+                "model": "decision",
+                "error": "",
+            },
+        ),
+        "failed": StoredTraceSignal(
+            trace_id="failed",
+            config_key=config.key(),
+            span_count=1,
+            classified_at=NOW,
+            data={"status": "failed", "scores": {}, "model": "decision", "error": "classification failed"},
+        ),
+        "stale": StoredTraceSignal(
+            trace_id="stale",
+            config_key="old-config",
+            span_count=1,
+            classified_at=NOW,
+            data={
+                "status": "classified",
+                "scores": {"user_frustration": 1.0},
+                "model": "old",
+                "error": "",
+            },
+        ),
+    }
+    database: Final = SignalStatusDatabase(config, rows)
+    monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=database))
+    viewer: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
+    request: Final = TraceFindingsRequest(
+        traces=tuple(
+            TraceIdentity(trace_id=trace_id) for trace_id in ("failed", "classified", "missing", "pending", "stale")
+        )
+    )
+
+    assert await get_signals(viewer) == config
+    results: Final = await trace_signal_statuses(request, viewer)
+
+    assert tuple((result.trace_id, result.status) for result in results) == (
+        ("failed", "failed"),
+        ("classified", "classified"),
+        ("missing", "unclassified"),
+        ("pending", "pending"),
+        ("stale", "unclassified"),
+    )
+    assert tuple((flag.signal_id, flag.name, flag.score) for flag in results[1].flags) == (
+        ("missing_capability", "Missing capability", 0.8),
+        ("user_frustration", "User frustration", 0.7),
+    )
+
+
+@pytest.mark.asyncio
+async def test_signal_endpoints_require_connected_postgres(monkeypatch: pytest.MonkeyPatch) -> None:
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setattr(proxy_server, "prisma_client", None)
+    auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
+
+    with pytest.raises(HTTPException) as error:
+        await get_signals(auth)
+
+    assert error.value.status_code == 503
+
+
+@pytest.mark.asyncio
+async def test_put_signals_saves_config_for_admin(monkeypatch: pytest.MonkeyPatch) -> None:
+    from litellm.proxy import proxy_server
+
+    database: Final = SignalStatusDatabase(SignalConfig(), {})
+    monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=database))
+    monkeypatch.setattr(proxy_server, "llm_router", signal_router())
+    auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
+    body: Final = SignalConfig(model="decision", threshold=0.7)
+
+    assert await put_signals(body, auth) == body
+
+    saved: Final = await database.saved.get()
+    assert saved[0] == "global"
+    assert isinstance(saved[1], str)
+    assert SignalConfig.model_validate_json(saved[1]) == body
+
+
+def test_signal_model_requires_a_ready_router() -> None:
+    with pytest.raises(HTTPException) as error:
+        validate_signal_model(SignalConfig(model="decision"), None)
+
+    assert error.value.status_code == 400
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY))
+async def test_put_signals_rejects_non_admin_roles(role: LitellmUserRoles) -> None:
+    auth: Final = UserAPIKeyAuth(user_role=role)
+    with pytest.raises(HTTPException) as error:
+        await put_signals(SignalConfig(model="decision"), auth)
+    assert error.value.status_code == 403
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model", ("chat", "unconfigured"))
+async def test_put_signals_rejects_chat_and_unknown_model_groups(monkeypatch: pytest.MonkeyPatch, model: str) -> None:
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setattr(proxy_server, "llm_router", signal_router())
+    auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
+
+    with pytest.raises(HTTPException) as error:
+        await put_signals(SignalConfig(model=model), auth)
+
+    assert error.value.status_code == 400
+    assert error.value.detail == "Choose a System 1 model (evaluation mode) configured on this proxy"
+
+
+def test_signal_model_accepts_only_evaluation_mode_groups() -> None:
+    assert validate_signal_model(SignalConfig(model="decision"), signal_router()) is None
+
+
 @pytest.mark.parametrize(
     "role",
     (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.TEAM),

tests/unit/proxy/lens/test_sources.py

@@ -4,13 +4,15 @@ from typing import Final, Literal
 
 import pytest
 
-from litellm.proxy.lens.models import Execution, ExecutionContent, MetadataFilter, Scope, TracePart
+from litellm.proxy.lens.models import Evidence, Execution, ExecutionContent, MetadataFilter, Scope, TracePart
 from litellm.proxy.lens.sources import SourceReader, execution_id, parse_execution
 from litellm.rust_bridge.trace.generated.models import (
     ActivityAvailability,
     AgentRow,
+    CountRow,
     ExecutionRow,
     LensContentParams,
+    LensEvidenceParams,
     PartRow,
 )
 from tests.unit.proxy.lens.test_state import lens
@@ -181,9 +183,17 @@ async def test_recorded_times_survive_source_catalog_reads_search_and_python(
 
     class ContentStorage:
         async def lens_content(self, parameters: LensContentParams) -> tuple[PartRow, ...]:
-            assert parameters.source == source and parameters.record_team == "team"
+            assert (
+                parameters.source == source
+                and parameters.record_team == "team"
+                and parameters.start_time == run.start_time
+            )
             return rows
 
+        async def lens_evidence(self, parameters: LensEvidenceParams) -> tuple[CountRow, ...]:
+            assert parameters.start_time == run.start_time
+            return (CountRow(count=1),)
+
     reader: Final = SourceReader(ContentStorage())
 
     async def read(identity: str, cursor: str, offset: int) -> ExecutionContent:
@@ -206,3 +216,8 @@ async def test_recorded_times_survive_source_catalog_reads_search_and_python(
     loaded: Final = await read(run.id, "", 1)
     assert loaded.parts == expected
     assert min(loaded.parts, key=lambda part: part.start_time).span_id == rows[-1].span_id
+    assert await reader.verify_evidence(
+        Scope(team_id="team"),
+        run,
+        Evidence(execution_id=run.id, span_id=rows[0].span_id, quote=rows[0].content),
+    )

tests/unit/rust_bridge/trace/test_queries.py

@@ -18,6 +18,7 @@ def test_named_query_rejects_offsets_outside_the_native_integer_range(offset: in
                 "source": "traces",
                 "id": "trace",
                 "record_team": "team",
+                "start_time": "",
                 "trace_ref": "ref",
                 "cursor": "",
                 "offset": offset,
@@ -34,6 +35,7 @@ def test_named_query_rejects_parameters_for_a_different_query() -> None:
         source="traces",
         id="trace",
         record_team="team",
+        start_time="",
         trace_ref="ref",
         cursor="",
         offset=0,

ui/litellm-dashboard/src/app/globals.css

@@ -211,6 +211,10 @@
   --trace-row-hover: oklch(0.975 0.008 215);
   --trace-row-selected: oklch(0.95 0.035 200);
   --trace-brand: oklch(0.6 0.13 195);
+  --finding-affected: var(--info);
+  --finding-unaffected: oklch(0.551 0.027 264.364 / 0.45);
+  --finding-quote: color-mix(in oklab, var(--warning) 16%, transparent);
+  --finding-ring: 0 0 0 1px oklch(0 0 0 / 0.06), 0 1px 2px -1px oklch(0 0 0 / 0.06), 0 2px 4px 0 oklch(0 0 0 / 0.04);
   --trace-border: oklch(0.92 0.01 230);
   --trace-line: oklch(0.88 0.03 205);
   --trace-card-border: oklch(0.93 0.01 230);
@@ -288,6 +292,10 @@
   --trace-row-hover: oklch(0.23 0.018 230);
   --trace-row-selected: oklch(0.29 0.05 210);
   --trace-brand: oklch(0.78 0.13 190);
+  --finding-affected: var(--info);
+  --finding-unaffected: oklch(0.707 0.022 261.325 / 0.35);
+  --finding-quote: color-mix(in oklab, var(--warning) 24%, transparent);
+  --finding-ring: 0 0 0 1px oklch(1 0 0 / 0.08);
   --trace-border: oklch(0.3 0.02 235);
   --trace-line: oklch(0.36 0.04 210);
   --trace-card-border: oklch(0.27 0.02 235);
@@ -326,6 +334,10 @@
   --color-trace-row-hover: var(--trace-row-hover);
   --color-trace-row-selected: var(--trace-row-selected);
   --color-trace-brand: var(--trace-brand);
+  --color-finding-affected: var(--finding-affected);
+  --color-finding-unaffected: var(--finding-unaffected);
+  --color-finding-quote: var(--finding-quote);
+  --shadow-finding-ring: var(--finding-ring);
   --color-trace-border: var(--trace-border);
   --color-trace-line: var(--trace-line);
   --color-trace-card-border: var(--trace-card-border);

ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx

@@ -141,9 +141,7 @@ describe("Lens interactive demo", () => {
     await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ }));
     const finding = screen.getByRole("complementary", { name: "Finding details" });
     expect(within(finding).getByText(/The support agent retries/)).toBeVisible();
-    const summaries = within(finding).getAllByText("support_agent", { exact: true });
-    await user.click(summaries[0]);
-    await user.click(within(finding).getAllByRole("button", { name: /Open original step/ })[0]);
+    await user.click(within(finding).getAllByRole("button", { name: "View span" })[0]);
     expect(await screen.findByRole("complementary", { name: "Span details" })).toHaveTextContent(
       "I will check that for you.",
     );
@@ -157,7 +155,8 @@ describe("Lens interactive demo", () => {
     await user.click(within(finding).getByRole("button", { name: "Back to finding" }));
     expect(within(finding).getByText(/The support agent retries/)).toBeVisible();
     await user.click(within(finding).getByRole("button", { name: "Close finding (Esc)" }));
-    expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible();
+    expect(await screen.findByRole("grid", { name: "Findings" })).toBeVisible();
+    expect(screen.queryByRole("complementary", { name: "Finding details" })).not.toBeInTheDocument();
     expect(network).not.toHaveBeenCalled();
     await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true"));
     await expectUrl(onUrlUpdate, (url) => expect(url.has("span")).toBe(false));
@@ -292,7 +291,7 @@ describe("Lens interactive demo", () => {
     await expectUrl(onUrlUpdate, (url) => expect(url.get("tab")).toBe("settings"));
     expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
     const panel = within(screen.getByRole("region", { name: "Settings" }));
-    expect(panel.getByRole("status")).toHaveTextContent("Tracing enabled");
+    expect(panel.getByText("Tracing enabled", { exact: true })).toBeVisible();
     expect(panel.getByRole("heading", { name: "Analysis worker" })).toBeVisible();
     expect(panel.getByRole("heading", { name: worker.name })).toBeVisible();
     expect(panel.getByText("Connected")).toBeVisible();

ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx

@@ -196,6 +196,7 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
                     readOnly={readOnly}
                     canMintTracingKey={isAdmin}
                     canViewFindings={canViewInvestigations}
+                    onSetUpSignals={canConfigure ? showSettings : undefined}
                   />
                 </TabsContent>
                 <TabsContent value="findings" className={PANEL}>

ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts

@@ -37,6 +37,8 @@ function demoLensApi(data: LensDemoData): LensApi {
     saveLens: readOnly,
     startRun: readOnly,
     watchAll: async () => ({ watching: [], skipped: [] }),
+    signalConfig: async () => ({ model: "", threshold: 0.5, signals: [] }),
+    saveSignalConfig: readOnly,
     cancelRun: readOnly,
     reviewFinding: readOnly,
     registerWorker: readOnly,
@@ -90,6 +92,8 @@ function demoTracesApi(data: LensDemoData): TracesApi {
         );
         return { ...trace, finding_count: assessed.length ? findings.size : null };
       }),
+    signals: async (traces) =>
+      traces.map((trace) => ({ ...trace, status: "unclassified" as const, flags: [], model: "", classified_at: null })),
     anyRecorded: async () => data.runs.length > 0,
     trace: (traceId) => found(run(traceId)?.trace),
     span: (traceId, spanId) => found(run(traceId)?.details.find((span) => span.span_id === spanId)),

ui/litellm-dashboard/src/components/lens/data/queries.ts

@@ -25,6 +25,7 @@ export const lensKeys = {
   models: (scope: string) => [...lensKeys.all, "models", { scope }] as const,
   modelDetails: (scope: string) => [...lensKeys.all, "model-details", { scope }] as const,
   activity: (scope: string) => [...lensKeys.all, "activity-available", { scope }] as const,
+  signalConfig: (scope: string) => [...lensKeys.all, "signal-config", { scope }] as const,
   discoveries: () => [...lensKeys.all, "discovery"] as const,
   discovery: (scope: string, source: Settings["source"], hours: number | undefined) =>
     [...lensKeys.discoveries(), { scope, source, hours }] as const,
@@ -46,6 +47,13 @@ export const lensQueries = {
   modelDetails(api: LensApi) {
     return queryOptions({ queryKey: lensKeys.modelDetails(api.scope), queryFn: () => api.modelDetails() });
   },
+  signalConfig(api: LensApi) {
+    return queryOptions({
+      queryKey: lensKeys.signalConfig(api.scope),
+      queryFn: () => api.signalConfig(),
+      staleTime: 5000,
+    });
+  },
   activity(api: LensApi, loaded: boolean) {
     const options = {
       queryKey: lensKeys.activity(api.scope),

ui/litellm-dashboard/src/components/lens/data/service.ts

@@ -13,6 +13,7 @@ import type {
   RunWindow,
   Sample,
   Settings,
+  SignalConfig,
   WorkerCreated,
 } from "../model/types";
 
@@ -62,6 +63,8 @@ export interface LensApi {
   saveLens(id: string | undefined, settings: Settings): Promise<Lens>;
   startRun(lensId: string, request?: RunWindow): Promise<void>;
   watchAll(): Promise<components["schemas"]["WatchAllResult"]>;
+  signalConfig(): Promise<SignalConfig>;
+  saveSignalConfig(config: SignalConfig): Promise<SignalConfig>;
   cancelRun(lensId: string): Promise<void>;
   reviewFinding(lensId: string, findingId: string, status: FindingStatus, reason: string): Promise<void>;
   registerWorker(analysisKeyId: string): Promise<WorkerCreated>;
@@ -168,6 +171,8 @@ export function liveLensApi(client: LensClient, apiClient: ApiClient, accessToke
       ),
     startRun: (lensId, request = {}) => sent(client.POST("/lens/{lens_id}/runs", { ...lens(lensId), body: request })),
     watchAll: () => required(client.POST("/lens/watch-all", { headers })),
+    signalConfig: () => required(client.GET("/lens/signals", { headers })),
+    saveSignalConfig: (config) => required(client.PUT("/lens/signals", { headers, body: config })),
     cancelRun: (lensId) => sent(client.POST("/lens/{lens_id}/cancel", lens(lensId))),
     reviewFinding: (lensId, findingId, status, reason) =>
       sent(

ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx

@@ -116,8 +116,7 @@ it("stacks a quote's original step over the finding and keeps the feedback draft
   const panel = screen.getByRole("complementary", { name: "Finding details" });
   const reason = () => within(panel).getByRole("textbox", { name: "What should Lens remember?", hidden: true });
   fireEvent.change(reason(), { target: { value: "Draft feedback" } });
-  for (const summary of within(panel).getAllByText(/quote$/)) await user.click(summary);
-  await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[0]);
+  await user.click(within(panel).getAllByRole("button", { name: "View span" })[0]);
   expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-1 at step-a");
   expect(reason()).not.toBeVisible();
   expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
@@ -127,13 +126,51 @@ it("stacks a quote's original step over the finding and keeps the feedback draft
   expect(reason()).toBeVisible();
   expect(reason()).toHaveValue("Draft feedback");
 
-  await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[1]);
+  await user.click(within(panel).getAllByRole("button", { name: "View span" })[1]);
   expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-2 at step-b");
   const url = new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? ""));
   expect(url.get("evidence")).toBe(traceOf("trace-2"));
   expect(url.get("evidence_span")).toBe("step-b");
 });
 
+it("reports how many sampled traces the finding affected and highlights each quoted line", () => {
+  const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id]));
+  const sampled = ["a", "b", "c", "d"].map((id) => ({
+    id: traceOf(id),
+    name: `run ${id}`,
+    start_time: "2026-10-01T10:00:00Z",
+    metadata: [],
+    root_seen: true,
+    service: "support_agent",
+    source: "traces" as const,
+    span_count: 1,
+    team_id: "",
+    trace_id: id,
+    trace_ref: "",
+  }));
+  const current: Finding = {
+    ...finding,
+    occurrences: [traceOf("a")],
+    evidence: [{ execution_id: traceOf("a"), span_id: "s", quote: "files:read is missing", role: "support" }],
+  };
+  renderWithLens(
+    <Inspector.Root
+      items={[]}
+      itemKey={ownedFindingKey}
+      selected={owned(current)}
+      onSelectedChange={vi.fn()}
+      noun="finding"
+      storageKey="test.finding"
+    >
+      <FindingPanel readOnly busy={false} sampledRuns={sampled} onReview={vi.fn()} />
+    </Inspector.Root>,
+  );
+  expect(screen.getByRole("region", { name: "Frequency" })).toHaveTextContent(/25%\s*1 of 4 traces affected/);
+  const example = screen.getByRole("article", { name: "run a" });
+  expect(within(example).getByText("files:read is missing").tagName).toBe("MARK");
+  expect(screen.queryByRole("article", { name: "run b" })).not.toBeInTheDocument();
+});
+
 it("shows contributing investigation runs and every affected trace, including older traces without retained quotes", async () => {
   const traceId = btoa(JSON.stringify(["traces", "", "older-trace", ""]));
   const current: Finding = {
@@ -142,8 +179,33 @@ it("shows contributing investigation runs and every affected trace, including ol
     investigation_runs: ["first-investigation-run", "second-investigation-run"],
   };
   renderWithLens(<Harness current={current} onReview={vi.fn()} />);
-  expect(screen.getByText("Found across 2 investigation runs")).toBeInTheDocument();
-  expect(screen.getByText(/1 affected trace/)).toBeInTheDocument();
-  fireEvent.click(screen.getByText("older-trace"));
-  expect(screen.getByRole("button", { name: "Open original trace" })).toBeInTheDocument();
+  expect(screen.getByText("1 affected trace")).toBeVisible();
+  expect(screen.getByText("Found across 2 investigation runs")).toBeVisible();
+  const example = screen.getByRole("article", { name: "Trace older-tr" });
+  expect(within(example).getByText("No quote was retained for this trace.")).toBeVisible();
+  expect(within(example).getByRole("button", { name: "View trace" })).toBeVisible();
+});
+
+it("shows the finding's priority and keeps the first three examples, revealing the rest on request", async () => {
+  const user = userEvent.setup();
+  const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id]));
+  const ids = ["t1", "t2", "t3", "t4", "t5"];
+  const current: Finding = {
+    ...finding,
+    occurrences: ids.map(traceOf),
+    evidence: ids.map((id) => ({
+      execution_id: traceOf(id),
+      span_id: id,
+      quote: `Input: ${id}\nOutput: done`,
+      role: "support" as const,
+    })),
+  };
+  renderWithLens(<Harness current={current} onReview={vi.fn()} />);
+  const panel = screen.getByRole("complementary", { name: "Finding details" });
+  expect(within(panel).getByText("High priority")).toBeVisible();
+  expect(within(panel).getAllByRole("article")).toHaveLength(3);
+  expect(within(panel).getAllByText("Call and result")).toHaveLength(3);
+  await user.click(within(panel).getByRole("button", { name: "Show 2 more examples" }));
+  expect(within(panel).getAllByRole("article")).toHaveLength(5);
+  expect(within(panel).queryByRole("button", { name: /Show \d+ more/ })).not.toBeInTheDocument();
 });

ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx

@@ -1,23 +1,32 @@
 "use client";
 
 import { useState } from "react";
-import { ArrowUpRight } from "lucide-react";
+import { ChevronRight, ClipboardCopy, X } from "lucide-react";
 
 import { Inspector } from "@/components/shared/Inspector";
 import { Button } from "@/components/ui/button";
 import { Textarea } from "@/components/ui/textarea";
+import { useNow } from "@/hooks/useNow";
+import { copyToClipboard } from "@/utils/dataUtils";
 
 import { AddToDatasetButton } from "../datasets/AddToDatasetDialog";
-import { evidenceTarget } from "../model/findings";
-import { runTime } from "../model/format";
+import { evidenceTarget, findingMarkdown } from "../model/findings";
+import { findingFrequency } from "../model/frequency";
+import { agoLabel, runTime } from "../model/format";
 import { findingAgents, findingKey, type OwnedFinding, sampledExecutions } from "../model/inbox";
 import type { Finding, Sample } from "../model/types";
 import { EvidenceView } from "./Evidence";
+import { FrequencyCard } from "./FrequencyCard";
 import { IssueBrief } from "./IssueBrief";
+import { PriorityPill } from "./PriorityMark";
 import { type EvidenceRef, useEvidenceRoute } from "../route";
 
 export const ownedFindingKey = (owned: OwnedFinding): string => findingKey(owned.lens, owned.finding);
 
+type Quote = Finding["evidence"][number];
+
+const SECTION_LABEL = "text-xs font-medium text-muted-foreground";
+
 export interface FindingDetailsProps {
   readonly finding: Finding;
   readonly lensId?: string;
@@ -27,6 +36,240 @@ export interface FindingDetailsProps {
   readonly busy: boolean;
   readonly onOpenEvidence: (evidence: EvidenceRef) => void;
   readonly onReview: (status: Finding["status"], reason: string) => void;
+  readonly onClose?: () => void;
+}
+
+function TopBar({ finding, onClose }: Pick<FindingDetailsProps, "finding" | "onClose">) {
+  const now = useNow(30000);
+  return (
+    <div className="sticky top-0 z-raised flex h-11 items-center justify-between gap-2 bg-background/95 px-4 backdrop-blur">
+      <p className="flex min-w-0 items-center gap-2 font-mono text-xs text-muted-foreground">
+        <span className="truncate" title={finding.id}>
+          {finding.id.slice(0, 8)}
+        </span>
+        <span aria-hidden="true">·</span>
+        <span className="whitespace-nowrap tabular-nums" title={runTime(finding.last_seen)}>
+          {agoLabel(Date.parse(finding.last_seen), now)}
+        </span>
+      </p>
+      <div className="flex shrink-0 gap-1.5">
+        <Button
+          variant="outline"
+          size="xs"
+          className="enabled:active:scale-[0.96]"
+          onClick={() => void copyToClipboard(findingMarkdown(finding), "Copied for agent")}
+        >
+          <ClipboardCopy />
+          Copy for agent
+        </Button>
+        {onClose && (
+          <Button
+            variant="ghost"
+            size="xs"
+            className="enabled:active:scale-[0.96]"
+            aria-label="Close finding (Esc)"
+            onClick={onClose}
+          >
+            <X />
+            Close
+          </Button>
+        )}
+      </div>
+    </div>
+  );
+}
+
+function Disclosure({ title, children }: { title: string; children: React.ReactNode }) {
+  return (
+    <details className="group rounded-lg">
+      <summary className="flex cursor-pointer list-none items-center gap-1.5 text-sm font-semibold [&::-webkit-details-marker]:hidden">
+        <ChevronRight
+          aria-hidden="true"
+          className="size-4 text-muted-foreground transition-[rotate] duration-150 group-open:rotate-90 motion-reduce:transition-none"
+        />
+        {title}
+      </summary>
+      <div className="mt-3">{children}</div>
+    </details>
+  );
+}
+
+function ProseSection({ title, children }: { title: string; children: string }) {
+  return (
+    <section>
+      <h2 className="mb-1.5 text-sm font-semibold">{title}</h2>
+      <p className="max-w-[70ch] text-sm leading-relaxed text-pretty whitespace-pre-wrap text-foreground/85">
+        {children}
+      </p>
+    </section>
+  );
+}
+
+const FIELD = /^(Input|Output|Status|Error)\s*:/gm;
+const FIELD_LABEL: Readonly<Record<string, string>> = {
+  "Input,Output": "Call and result",
+  Input: "Call input",
+  Output: "Returned output",
+  Status: "Span status",
+  Error: "Error",
+};
+
+function quoteLabel(quote: Quote, isTrace: boolean): string {
+  if (quote.role === "counterexample") return "Counterexample";
+  const fields = [...new Set(Array.from(quote.quote.matchAll(FIELD), (m) => m[1]))].join(",");
+  return FIELD_LABEL[fields] ?? (isTrace ? "Trace step" : "Logged request");
+}
+
+const MARK = {
+  support: "rounded-sm bg-finding-quote px-0.5 text-inherit",
+  counterexample: "rounded-sm bg-success/20 px-0.5 text-inherit",
+} as const;
+
+function QuoteCard({ quote, onOpen }: { quote: Quote; onOpen: () => void }) {
+  const isTrace = evidenceTarget(quote.execution_id)?.source === "traces";
+  return (
+    <div className="-mx-1 rounded-lg border border-transparent bg-background p-2.5 shadow-finding-ring">
+      <div className="mb-1.5 flex items-center justify-between gap-2">
+        <span className="min-w-0 truncate text-sm font-medium">{quoteLabel(quote, isTrace)}</span>
+        <Button variant="outline" size="xs" className="enabled:active:scale-[0.96]" onClick={onOpen}>
+          {isTrace ? "View span" : "View request"}
+        </Button>
+      </div>
+      <button
+        type="button"
+        tabIndex={-1}
+        title="Open this evidence"
+        onClick={onOpen}
+        className="block w-full overflow-x-auto rounded-md bg-muted/60 py-1.5 text-left font-mono text-xs leading-5 transition-[background-color] duration-150 hover:bg-muted"
+      >
+        <code className="block border-l-2 border-warning/40 px-2.5 break-words whitespace-pre-wrap text-foreground/85">
+          <mark className={MARK[quote.role]}>{quote.quote}</mark>
+        </code>
+      </button>
+    </div>
+  );
+}
+
+function EvidenceRail({ children }: { children: React.ReactNode }) {
+  return (
+    <>
+      <div className="relative flex gap-2">
+        <div className="relative flex w-3 shrink-0 flex-col items-center">
+          <div className="flex h-5 w-full items-center justify-center">
+            <div className="size-1 rounded-full bg-border" />
+          </div>
+          <div aria-hidden="true" className="h-2 w-px bg-border" />
+        </div>
+        <div className="min-w-0 flex-1 text-xs leading-5 font-medium text-muted-foreground">Evidence</div>
+      </div>
+      <div className="relative">
+        <div aria-hidden="true" className="absolute inset-y-0 left-[5px] w-px bg-border" />
+        <div className="relative z-raised space-y-3">{children}</div>
+      </div>
+    </>
+  );
+}
+
+interface ExampleGroup {
+  readonly id: string;
+  readonly run: Sample["executions"][number] | undefined;
+  readonly quotes: readonly Quote[];
+}
+
+function Example({ group, onOpenEvidence }: { group: ExampleGroup; onOpenEvidence: (e: EvidenceRef) => void }) {
+  const traceId = evidenceTarget(group.id)?.id;
+  const name = group.run?.name ?? (traceId ? `Trace ${traceId.slice(0, 8)}` : "Recorded run");
+  return (
+    <article aria-label={name} className="flex w-full flex-col items-start gap-3 rounded-xl bg-muted/50 p-3.5">
+      <div className="flex w-full items-baseline justify-between gap-3">
+        <h3 className="min-w-0 truncate text-sm font-medium" title={traceId}>
+          {name}
+        </h3>
+        <span className="shrink-0 text-xs text-muted-foreground tabular-nums">
+          {[group.run?.service, group.run && runTime(group.run.start_time)].filter(Boolean).join(" · ")}
+        </span>
+      </div>
+      <div className="w-full">
+        {group.quotes.length === 0 ? (
+          <div className="flex items-center justify-between gap-3">
+            <p className="text-sm text-muted-foreground">No quote was retained for this trace.</p>
+            <Button variant="outline" size="xs" onClick={() => onOpenEvidence({ id: group.id, span: "" })}>
+              View trace
+            </Button>
+          </div>
+        ) : (
+          <EvidenceRail>
+            {group.quotes.map((quote, i) => (
+              <QuoteCard
+                key={`${quote.span_id}-${i}`}
+                quote={quote}
+                onOpen={() => onOpenEvidence({ id: quote.execution_id, span: quote.span_id })}
+              />
+            ))}
+          </EvidenceRail>
+        )}
+      </div>
+    </article>
+  );
+}
+
+const VISIBLE_EXAMPLES = 3;
+
+function Examples({
+  groups,
+  onOpenEvidence,
+}: {
+  groups: readonly ExampleGroup[];
+  onOpenEvidence: (e: EvidenceRef) => void;
+}) {
+  const [expanded, setExpanded] = useState(false);
+  if (groups.length === 0) return <p className="text-sm text-muted-foreground">No examples were recorded.</p>;
+  const shown = expanded ? groups : groups.slice(0, VISIBLE_EXAMPLES);
+  const hidden = groups.length - shown.length;
+  return (
+    <>
+      {shown.map((group) => (
+        <Example key={group.id} group={group} onOpenEvidence={onOpenEvidence} />
+      ))}
+      {hidden > 0 && (
+        <Button
+          variant="ghost"
+          size="sm"
+          className="self-start text-muted-foreground"
+          onClick={() => setExpanded(true)}
+        >
+          Show {hidden} more {hidden === 1 ? "example" : "examples"}
+        </Button>
+      )}
+    </>
+  );
+}
+
+function ReviewForm({ finding, busy, onReview }: Pick<FindingDetailsProps, "finding" | "busy" | "onReview">) {
+  const [reason, setReason] = useState(finding.reason ?? "");
+  return (
+    <section className="space-y-3 border-t pt-6">
+      <label className="grid gap-2 text-sm font-medium">
+        What should Lens remember?
+        <Textarea
+          value={reason}
+          onChange={(e) => setReason(e.target.value)}
+          placeholder="What should Lens know about this finding?"
+        />
+      </label>
+      <p className="text-xs text-muted-foreground">Your explanation informs future scans of this Lens.</p>
+      <div className="flex flex-wrap gap-2">
+        {finding.kind === "issue" && (
+          <Button disabled={busy} onClick={() => onReview(finding.status === "resolved" ? "open" : "resolved", reason)}>
+            {finding.status === "resolved" ? "Reopen" : "Mark resolved"}
+          </Button>
+        )}
+        <Button disabled={busy} variant="outline" onClick={() => onReview("dismissed", reason)}>
+          This is expected
+        </Button>
+      </div>
+    </section>
+  );
 }
 
 export function FindingDetails({
@@ -38,132 +281,72 @@ export function FindingDetails({
   busy,
   onOpenEvidence,
   onReview,
+  onClose,
 }: FindingDetailsProps) {
-  const [reason, setReason] = useState(finding.reason ?? "");
-  const evidenceGroups = [...new Set([...finding.occurrences, ...finding.evidence.map((e) => e.execution_id)])].map(
-    (id) => ({
-      id,
-      run: sampledRuns.find((r) => r.id === id),
-      quotes: finding.evidence.filter((e) => e.execution_id === id),
-    }),
-  );
+  const groups: ExampleGroup[] = [
+    ...new Set([...finding.evidence.map((e) => e.execution_id), ...finding.occurrences]),
+  ].map((id) => ({
+    id,
+    run: sampledRuns.find((r) => r.id === id),
+    quotes: finding.evidence.filter((e) => e.execution_id === id),
+  }));
+  const affected = finding.occurrences.length;
+  const runs = finding.investigation_runs?.length ?? 0;
   return (
     <div className="min-h-0 flex-1 overflow-y-auto">
-      <header className="flex flex-col gap-1.5 border-b px-4 py-4">
-        <h2 className="text-xl leading-snug font-semibold">{finding.title}</h2>
-        <p className="text-sm text-muted-foreground">
-          {agents.length > 0 && <span className="font-medium text-foreground">{agents.join(", ")} · </span>}
-          {finding.kind === "issue" ? `${finding.priority} priority` : "Pattern"} · {finding.occurrences?.length ?? 0}{" "}
-          affected {finding.occurrences?.length === 1 ? "trace" : "traces"}
-        </p>
-        {(finding.investigation_runs?.length ?? 0) > 0 && (
-          <p className="text-sm text-muted-foreground">
-            Found across {finding.investigation_runs.length} investigation{" "}
-            {finding.investigation_runs.length === 1 ? "run" : "runs"}
-          </p>
-        )}
-        {lensId && finding.evidence.length > 0 && (
-          <AddToDatasetButton
-            sources={[{ kind: "finding", lens_id: lensId, finding_ids: [finding.id] }]}
-            agentName={agents[0]}
-            label="Add evidence to dataset"
-            className="mt-1 self-start"
-          />
-        )}
-      </header>
-      <div className="space-y-6 p-4">
-        {finding.brief ? (
-          <IssueBrief title={finding.title} brief={finding.brief} />
-        ) : (
-          <>
-            <div>
-              <p className="mb-2 text-sm font-medium">What happened</p>
-              <p className="text-sm leading-6 whitespace-pre-wrap">{finding.description}</p>
-            </div>
-            {finding.suggestion && (
-              <div className="border-y py-4">
-                <p className="text-sm font-medium">What to do next</p>
-                <p className="mt-2 text-sm leading-6">{finding.suggestion}</p>
-              </div>
+      <TopBar finding={finding} onClose={onClose} />
+      <div className="mx-auto flex w-full max-w-3xl flex-col gap-7 px-6 pt-4 pb-16">
+        <header className="flex flex-col gap-3">
+          <h1 className="text-2xl leading-tight font-semibold tracking-tight text-balance">{finding.title}</h1>
+          <div className="flex flex-wrap items-center gap-x-3 gap-y-1.5 text-xs text-muted-foreground">
+            {finding.kind === "issue" ? (
+              <PriorityPill priority={finding.priority} />
+            ) : (
+              <span className="inline-flex h-5 items-center rounded-full bg-muted px-2 font-medium">Pattern</span>
             )}
-          </>
+            {agents.length > 0 && <span className="font-medium text-foreground">{agents.join(", ")}</span>}
+            <span className="tabular-nums">
+              {affected} affected {affected === 1 ? "trace" : "traces"}
+            </span>
+            {runs > 0 && (
+              <span className="tabular-nums">
+                Found across {runs} investigation {runs === 1 ? "run" : "runs"}
+              </span>
+            )}
+          </div>
+        </header>
+        <ProseSection title="Summary">{finding.description}</ProseSection>
+        {finding.suggestion && <ProseSection title="Suggested fix">{finding.suggestion}</ProseSection>}
+        {finding.brief && (
+          <Disclosure title="Issue brief and test cases">
+            <IssueBrief title={finding.title} brief={finding.brief} />
+          </Disclosure>
         )}
         {finding.limitation && (
-          <details className="text-sm">
-            <summary className="cursor-pointer font-medium">Evidence limits</summary>
-            <p className="mt-3 leading-6 text-muted-foreground">{finding.limitation}</p>
-          </details>
+          <Disclosure title="Evidence limits">
+            <p className="max-w-[70ch] text-sm leading-relaxed text-pretty text-muted-foreground">
+              {finding.limitation}
+            </p>
+          </Disclosure>
         )}
-        <div>
-          <p className="text-sm font-medium">Affected traces and counterexamples</p>
-          <p className="mt-1 mb-3 text-xs text-muted-foreground">
-            Exact quotes from the recorded activity. Counterexamples are labeled separately from supporting evidence.
-          </p>
-          <div className="space-y-2">
-            {evidenceGroups.map((group) => (
-              <details key={group.id} className="rounded-lg border p-3">
-                <summary className="cursor-pointer text-sm font-medium">
-                  {group.run?.name ?? evidenceTarget(group.id)?.id.slice(0, 12) ?? "Recorded run"}
-                  <span className="ml-2 text-xs font-normal text-muted-foreground">
-                    {group.quotes.length} {group.quotes.length === 1 ? "quote" : "quotes"}
-                    {group.run ? ` · ${runTime(group.run.start_time)}` : ""}
-                  </span>
-                </summary>
-                <div className="mt-3 space-y-3">
-                  {group.quotes.length === 0 && (
-                    <Button variant="ghost" size="sm" onClick={() => onOpenEvidence({ id: group.id, span: "" })}>
-                      Open original trace
-                      <ArrowUpRight className="size-3" />
-                    </Button>
-                  )}
-                  {group.quotes.map((e, i) => (
-                    <div key={`${e.span_id}-${i}`} className="rounded-md bg-muted/40 p-3">
-                      {e.role === "counterexample" && (
-                        <p className="mb-1 text-xs font-medium text-muted-foreground">Counterexample</p>
-                      )}
-                      <blockquote className="text-xs leading-5 whitespace-pre-wrap break-words">{e.quote}</blockquote>
-                      <Button
-                        variant="ghost"
-                        size="sm"
-                        className="mt-2"
-                        onClick={() => onOpenEvidence({ id: e.execution_id, span: e.span_id })}
-                      >
-                        {evidenceTarget(e.execution_id)?.source === "traces" ? "Open original step" : "Open request"}
-                        <ArrowUpRight className="size-3" />
-                      </Button>
-                    </div>
-                  ))}
-                </div>
-              </details>
-            ))}
-          </div>
-        </div>
-        {!readOnly && (
-          <div className="space-y-3 border-t pt-4">
-            <label className="grid gap-2 text-sm">
-              What should Lens remember?
-              <Textarea
-                value={reason}
-                onChange={(e) => setReason(e.target.value)}
-                placeholder="What should Lens know about this finding?"
+        <section>
+          <h2 className={`mb-2 ${SECTION_LABEL}`}>Monitors</h2>
+          <FrequencyCard frequency={findingFrequency(finding.occurrences, sampledRuns)} />
+        </section>
+        <section className="flex flex-col gap-2">
+          <div className="flex items-center justify-between gap-2">
+            <h2 className={SECTION_LABEL}>Examples</h2>
+            {lensId && finding.evidence.length > 0 && (
+              <AddToDatasetButton
+                sources={[{ kind: "finding", lens_id: lensId, finding_ids: [finding.id] }]}
+                agentName={agents[0]}
+                label="Add evidence to dataset"
               />
-            </label>
-            <p className="text-xs text-muted-foreground">Your explanation informs future scans of this Lens.</p>
-            <div className="flex flex-wrap gap-2">
-              {finding.kind === "issue" && (
-                <Button
-                  disabled={busy}
-                  onClick={() => onReview(finding.status === "resolved" ? "open" : "resolved", reason)}
-                >
-                  {finding.status === "resolved" ? "Reopen" : "Mark resolved"}
-                </Button>
-              )}
-              <Button disabled={busy} variant="outline" onClick={() => onReview("dismissed", reason)}>
-                This is expected
-              </Button>
-            </div>
+            )}
           </div>
-        )}
+          <Examples groups={groups} onOpenEvidence={onOpenEvidence} />
+        </section>
+        {!readOnly && <ReviewForm finding={finding} busy={busy} onReview={onReview} />}
       </div>
     </div>
   );

ui/litellm-dashboard/src/components/lens/investigations/FindingsView.integration.test.tsx

@@ -29,13 +29,9 @@ it("deduplicates findings across investigations and applies feedback to every so
   renderWithLens(<FindingsView />, { searchParams: "?tab=findings", onUrlUpdate });
   const rows = await screen.findAllByRole("row", { name: issue.title });
   expect(rows).toHaveLength(1);
-  expect(screen.getByRole("columnheader", { name: "Investigation", exact: true })).toBeVisible();
-  const investigations = `${support.settings.name}, ${twin.settings.name}`;
-  expect(within(rows[0]).getByRole("cell", { name: investigations, exact: true })).toHaveAttribute(
-    "title",
-    investigations,
-  );
-  expect(within(rows[0]).getByRole("cell", { name: "2", exact: true })).toBeVisible();
+  expect(
+    within(rows[0]).getByTitle(`2 affected traces across ${support.settings.name}, ${twin.settings.name}`),
+  ).toBeVisible();
   await user.click(rows[0]);
   const panel = await screen.findByRole("complementary", { name: "Finding details" });
   fireEvent.change(within(panel).getByRole("textbox"), { target: { value: "A handoff now handles failures" } });
@@ -107,7 +103,7 @@ it("reviews only the selected check when two findings have the same title", asyn
   renderWithLens(<FindingsView />, { searchParams: "?tab=findings" });
   const rows = await screen.findAllByRole("row", { name: issue.title });
   expect(rows).toHaveLength(2);
-  expect(within(rows[0]).getByRole("cell", { name: support.settings.name, exact: true })).toBeVisible();
+  expect(within(rows[0]).getByTitle(new RegExp(`across ${support.settings.name}$`))).toBeVisible();
   await user.click(rows[0]);
   await user.click(await screen.findByRole("button", { name: "Mark resolved" }));
   await waitFor(() => expect(proxy.patch).toHaveBeenCalledTimes(1));
@@ -122,3 +118,38 @@ it("opens a grouped finding from a link to any of its owning investigations", as
   expect(within(panel).getByRole("heading", { name: issue.title })).toBeVisible();
   expect(screen.getByRole("row", { name: issue.title })).toHaveAttribute("aria-selected", "true");
 });
+
+it("ranks findings under high, medium and low priority headings with the highest first", async () => {
+  const at = (id: string, priority: "high" | "medium" | "low", last_seen: string) => ({
+    ...issue,
+    id,
+    check_id: id,
+    title: `${priority} ${id}`,
+    priority,
+    last_seen,
+  });
+  const findings = [
+    at("newest-low", "low", "2026-10-06T00:00:00Z"),
+    at("old-high", "high", "2026-09-01T00:00:00Z"),
+    at("medium", "medium", "2026-10-05T00:00:00Z"),
+    at("new-high", "high", "2026-10-04T00:00:00Z"),
+  ];
+  proxy.get.mockImplementation(async (path) =>
+    path === "/lens" ? { lenses: [{ ...support, findings }], workers: [], tracing_enabled: true } : { data: [] },
+  );
+  renderWithLens(<FindingsView readOnly />, { searchParams: "?tab=findings" });
+  const groups = await screen.findAllByRole("rowgroup");
+  expect(groups.map((group) => group.getAttribute("aria-label"))).toEqual([
+    "High priority findings",
+    "Medium priority findings",
+    "Low priority findings",
+  ]);
+  const titles = (group: HTMLElement) =>
+    within(group)
+      .getAllByRole("row")
+      .map((row) => row.getAttribute("aria-label"))
+      .filter(Boolean);
+  expect(titles(groups[0])).toEqual(["high new-high", "high old-high"]);
+  expect(groups[0]).toHaveTextContent(/High priority\s*2/);
+  expect(titles(groups[2])).toEqual(["low newest-low"]);
+});

ui/litellm-dashboard/src/components/lens/investigations/FindingsView.tsx

@@ -1,16 +1,15 @@
 "use client";
 
 import { useQuery } from "@tanstack/react-query";
-import { ChevronRight } from "lucide-react";
-import { Inspector } from "@/components/shared/Inspector";
+import { Inspector, useInspector } from "@/components/shared/Inspector";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { useNow } from "@/hooks/useNow";
-import { cn } from "@/lib/cva.config";
 import { formatActivityTimestamp } from "@/utils/activityTimestamp";
 import { useLensApi } from "../data/LensServices";
 import { useLensUpdate } from "../data/mutations";
 import { lensQueries } from "../data/queries";
 import { agoLabel } from "../model/format";
+import { findingFrequency, percentLabel } from "../model/frequency";
 import {
   ALL_AGENTS,
   filterInbox,
@@ -18,7 +17,7 @@ import {
   inboxAgents,
   inboxFinding,
   inboxRows,
-  sampledExecutions,
+  inboxSampledRuns,
   type InboxRow,
   type Priority,
 } from "../model/inbox";
@@ -28,8 +27,8 @@ import { FINDING_PANEL_WIDTH_KEY } from "../storage";
 import { EvidenceView } from "./Evidence";
 import { FindingDetails } from "./FindingDetails";
 import { InvestigationError, InvestigationsLoading } from "./InvestigationStates";
+import { PRIORITY_LABEL, PRIORITY_ORDER, PriorityDot } from "./PriorityMark";
 
-const PRIORITY_DOT = { high: "bg-destructive", medium: "bg-warning", low: "bg-muted-foreground/50" } as const;
 const PRIORITIES: { value: Priority | "all"; label: string }[] = [
   { value: "all", label: "All priorities" },
   { value: "high", label: "High" },
@@ -37,15 +36,6 @@ const PRIORITIES: { value: Priority | "all"; label: string }[] = [
   { value: "low", label: "Low" },
 ];
 
-function InvestigationCell({ sources }: Pick<InboxRow, "sources">) {
-  const names = [...new Set(sources.map(({ lens }) => lens.settings.name))].join(", ");
-  return (
-    <td className="hidden truncate px-3 text-muted-foreground lg:table-cell" title={names}>
-      {names}
-    </td>
-  );
-}
-
 function FilterSelect<T extends string>({
   label,
   value,
@@ -59,7 +49,11 @@ function FilterSelect<T extends string>({
 }) {
   return (
     <Select items={items} value={value} onValueChange={(next: T | null) => next !== null && onChange(next)}>
-      <SelectTrigger size="sm" className="h-7 min-w-32 text-xs" aria-label={label}>
+      <SelectTrigger
+        size="sm"
+        className="h-7 min-w-0 flex-1 border-transparent bg-muted/60 text-xs shadow-none hover:bg-muted"
+        aria-label={label}
+      >
         <SelectValue />
       </SelectTrigger>
       <SelectContent>
@@ -73,6 +67,67 @@ function FilterSelect<T extends string>({
   );
 }
 
+function FindingRow({ row, now }: { row: InboxRow; now: number }) {
+  const frequency = findingFrequency(inboxFinding(row).occurrences, inboxSampledRuns(row));
+  const percent = percentLabel(frequency.affected, frequency.total);
+  return (
+    <Inspector.Row
+      item={row}
+      render={
+        <div
+          role="row"
+          tabIndex={0}
+          aria-label={row.title}
+          className="mx-2 block cursor-pointer space-y-1 rounded-md px-2 py-2 transition-[background-color] duration-150 outline-none hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-ring/50 data-[state=selected]:bg-muted"
+        />
+      }
+    >
+      <div role="gridcell" className="line-clamp-2 text-xs leading-snug text-pretty text-foreground">
+        {row.title}
+      </div>
+      <div className="flex flex-wrap items-center justify-between gap-x-2 gap-y-1 text-xs text-muted-foreground">
+        <span className="tabular-nums" title={formatActivityTimestamp(row.lastSeen)}>
+          {agoLabel(Date.parse(row.lastSeen), now)}
+        </span>
+        <span
+          className="whitespace-nowrap text-foreground/75 tabular-nums"
+          title={`${row.runs} affected ${row.runs === 1 ? "trace" : "traces"} across ${row.sources.map(({ lens }) => lens.settings.name).join(", ")}`}
+        >
+          {percent ? `${percent} affected` : `${row.runs} ${row.runs === 1 ? "trace" : "traces"}`}
+        </span>
+      </div>
+    </Inspector.Row>
+  );
+}
+
+function FindingList({ rows, now }: { rows: readonly InboxRow[]; now: number }) {
+  const groups = PRIORITY_ORDER.map((priority) => ({
+    priority,
+    rows: rows.filter((row) => row.priority === priority),
+  })).filter((group) => group.rows.length > 0);
+  return (
+    <div role="grid" aria-label="Findings" className="min-h-0 flex-1 overflow-y-auto overscroll-contain pb-2">
+      {groups.map((group) => (
+        <div role="rowgroup" key={group.priority} aria-label={`${PRIORITY_LABEL[group.priority]} priority findings`}>
+          <div
+            role="row"
+            className="sticky top-0 z-raised flex items-center gap-2 bg-background/95 px-4 pt-3 pb-1.5 text-xs font-medium text-muted-foreground backdrop-blur"
+          >
+            <PriorityDot priority={group.priority} />
+            <span role="columnheader">{PRIORITY_LABEL[group.priority]} priority</span>
+            <span className="ml-auto tabular-nums">{group.rows.length}</span>
+          </div>
+          <div className="space-y-0.5">
+            {group.rows.map((row) => (
+              <FindingRow key={row.key} row={row} now={now} />
+            ))}
+          </div>
+        </div>
+      ))}
+    </div>
+  );
+}
+
 function InboxDetail({
   row,
   readOnly,
@@ -84,6 +139,7 @@ function InboxDetail({
   busy: boolean;
   onReview: (row: InboxRow, status: Finding["status"], reason: string) => void;
 }) {
+  const { close } = useInspector<InboxRow>();
   const { evidence, setEvidence } = useEvidenceRoute();
   const owner =
     row.sources.find(({ finding }) => finding.evidence.some((quote) => quote.execution_id === evidence?.id)) ??
@@ -95,11 +151,12 @@ function InboxDetail({
           key={row.key}
           finding={inboxFinding(row)}
           agents={row.agents}
-          sampledRuns={row.sources.flatMap(({ lens }) => sampledExecutions(lens))}
+          sampledRuns={inboxSampledRuns(row)}
           readOnly={readOnly}
           busy={busy}
           onOpenEvidence={setEvidence}
           onReview={(status, reason) => onReview(row, status, reason)}
+          onClose={close}
         />
       </div>
       {evidence && (
@@ -151,7 +208,7 @@ export function FindingsView({ readOnly = false }: { readOnly?: boolean }) {
       noun="finding"
       storageKey={FINDING_PANEL_WIDTH_KEY}
     >
-      <section aria-label="Findings" className="flex min-h-0 flex-1 flex-col">
+      <div className="flex min-h-0 flex-1 flex-col overflow-hidden rounded-lg border bg-background">
         {(list.error || update.error) && (
           <InvestigationError
             message={(list.error ?? update.error)!.message}
@@ -161,92 +218,48 @@ export function FindingsView({ readOnly = false }: { readOnly?: boolean }) {
             }}
           />
         )}
-        <div className="flex min-h-10 shrink-0 flex-wrap items-center gap-2 border-b px-3 py-1.5">
-          <FilterSelect label="Filter by agent" value={filters.agent} items={agents} onChange={filters.setAgent} />
-          <FilterSelect
-            label="Filter by priority"
-            value={filters.priority}
-            items={PRIORITIES}
-            onChange={filters.setPriority}
-          />
-        </div>
-        <div className="min-h-0 flex-1 overflow-auto">
-          <table aria-label="Findings" className="w-full table-fixed border-collapse text-left text-xs">
-            <thead className="sticky top-0 z-sticky bg-muted/40 backdrop-blur">
-              <tr className="h-8 border-b text-xs tracking-wider text-muted-foreground uppercase">
-                <th className="w-20 px-3 font-medium">Priority</th>
-                <th className="px-3 font-medium">Finding</th>
-                <th className="hidden w-48 px-3 font-medium lg:table-cell">Investigation</th>
-                <th className="hidden w-40 px-3 font-medium md:table-cell">Agent</th>
-                <th className="hidden w-16 px-3 text-right font-medium sm:table-cell">Traces</th>
-                <th className="hidden w-24 px-3 font-medium lg:table-cell">Last seen</th>
-                <th className="w-7">
-                  <span className="sr-only">Details</span>
-                </th>
-              </tr>
-            </thead>
-            <tbody>
-              {rows.map((row) => (
-                <Inspector.Row
-                  key={row.key}
-                  item={row}
-                  render={
-                    <tr
-                      tabIndex={0}
-                      aria-label={row.title}
-                      className="h-9 cursor-pointer border-b border-border/60 hover:bg-trace-row-hover focus-visible:outline-2 focus-visible:outline-ring data-[state=selected]:bg-trace-row-selected"
-                    />
-                  }
-                >
-                  <td className="px-3">
-                    <span className="inline-flex items-center gap-1.5 text-muted-foreground">
-                      <span aria-hidden="true" className={cn("size-1.5 rounded-full", PRIORITY_DOT[row.priority])} />
-                      {row.priority}
-                    </span>
-                  </td>
-                  <td className="px-3 py-2 sm:py-0" title={row.suggestion || undefined}>
-                    <span className="line-clamp-2 text-foreground sm:block sm:truncate">{row.title}</span>
-                    <span className="mt-1 block text-xs text-muted-foreground md:hidden">
-                      {row.agents.join(", ")} · {row.runs} {row.runs === 1 ? "trace" : "traces"}
-                    </span>
-                  </td>
-                  <InvestigationCell sources={row.sources} />
-                  <td
-                    className="hidden truncate px-3 text-muted-foreground md:table-cell"
-                    title={row.agents.join(", ")}
-                  >
-                    {row.agents.join(", ")}
-                  </td>
-                  <td className="hidden px-3 text-right font-mono tabular-nums sm:table-cell">{row.runs}</td>
-                  <td
-                    className="hidden px-3 tabular-nums text-muted-foreground lg:table-cell"
-                    title={formatActivityTimestamp(row.lastSeen)}
-                  >
-                    {agoLabel(Date.parse(row.lastSeen), now)}
-                  </td>
-                  <td>
-                    <ChevronRight aria-hidden="true" className="size-3 text-muted-foreground/60" />
-                  </td>
-                </Inspector.Row>
-              ))}
-            </tbody>
-          </table>
-          {!rows.length && !list.error && (
-            <p className="px-4 py-16 text-center text-xs text-muted-foreground">
-              {all.length
-                ? "No findings match these filters."
-                : "No open findings yet. New problems show up here as soon as an investigation spots them."}
-            </p>
+        <div className="flex min-h-0 flex-1">
+          <section
+            aria-label="Findings list"
+            className={`flex min-h-0 w-full flex-col border-r md:w-[22rem] md:shrink-0 lg:w-[28rem] ${selected ? "hidden md:flex" : "flex"}`}
+          >
+            <div className="flex shrink-0 items-center gap-2 px-3 py-2">
+              <FilterSelect label="Filter by agent" value={filters.agent} items={agents} onChange={filters.setAgent} />
+              <FilterSelect
+                label="Filter by priority"
+                value={filters.priority}
+                items={PRIORITIES}
+                onChange={filters.setPriority}
+              />
+            </div>
+            <FindingList rows={rows} now={now} />
+            {!rows.length && !list.error && (
+              <p className="px-4 py-16 text-center text-xs text-muted-foreground">
+                {all.length
+                  ? "No findings match these filters."
+                  : "No open findings yet. New problems show up here as soon as an investigation spots them."}
+              </p>
+            )}
+            <footer className="flex h-8 shrink-0 items-center border-t px-3 text-xs text-muted-foreground">
+              {rows.length} {rows.length === 1 ? "finding" : "findings"}
+              {rows.length !== all.length && ` of ${all.length}`}
+            </footer>
+          </section>
+          {selected ? (
+            <aside
+              aria-label="Finding details"
+              data-testid="finding-panel"
+              className="flex min-h-0 min-w-0 flex-1 flex-col"
+            >
+              <InboxDetail row={selected} readOnly={readOnly} busy={update.isPending} onReview={review} />
+            </aside>
+          ) : (
+            <div className="hidden min-w-0 flex-1 items-center justify-center p-8 text-sm text-muted-foreground md:flex">
+              {rows.length ? "Select a finding to see how often it happens and where." : null}
+            </div>
           )}
         </div>
-        <footer className="flex h-8 shrink-0 items-center border-t bg-muted/30 px-3 text-xs text-muted-foreground">
-          {rows.length} {rows.length === 1 ? "finding" : "findings"}
-          {rows.length !== all.length && ` of ${all.length}`}
-        </footer>
-      </section>
-      <Inspector.Panel label="Finding details" testId="finding-panel">
-        {(row: InboxRow) => <InboxDetail row={row} readOnly={readOnly} busy={update.isPending} onReview={review} />}
-      </Inspector.Panel>
+      </div>
     </Inspector.Root>
   );
 }

ui/litellm-dashboard/src/components/lens/investigations/FrequencyCard.tsx

@@ -0,0 +1,88 @@
+"use client";
+
+import { Bar, BarChart, CartesianGrid, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts";
+import { dayLabel, type Frequency, percentLabel } from "../model/frequency";
+
+const AFFECTED = "var(--finding-affected)";
+const UNAFFECTED = "var(--finding-unaffected)";
+const TICK = { fontSize: 11, fill: "var(--muted-foreground)" };
+const TOP_RADIUS: [number, number, number, number] = [3, 3, 0, 0];
+
+function Legend() {
+  return (
+    <div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
+      <span className="flex items-center gap-1.5">
+        <span aria-hidden="true" className="size-2 rounded-sm bg-finding-affected" />
+        Affected
+      </span>
+      <span className="flex items-center gap-1.5">
+        <span aria-hidden="true" className="size-2 rounded-sm bg-finding-unaffected" />
+        Unaffected
+      </span>
+    </div>
+  );
+}
+
+export function FrequencyCard({ frequency }: { frequency: Frequency }) {
+  const percent = percentLabel(frequency.affected, frequency.total);
+  const data = frequency.days.map((d) => ({ ...d, label: dayLabel(d.day) }));
+  const range = data.length > 0 ? `${data[0].label} – ${data[data.length - 1].label}` : null;
+  return (
+    <section
+      aria-label="Frequency"
+      className="space-y-4 rounded-xl border border-transparent bg-muted/40 p-4 shadow-finding-ring"
+    >
+      <div aria-live="polite" className="flex items-start justify-between gap-4">
+        <div>
+          <h3 className="mb-1 text-sm font-medium">Frequency</h3>
+          <p className="text-2xl leading-tight font-semibold tracking-tight tabular-nums">
+            {percent ?? "—"}
+            <span className="ml-1.5 text-sm font-normal tracking-normal text-muted-foreground">
+              {frequency.affected} of {frequency.total} traces affected
+            </span>
+          </p>
+        </div>
+        {range && <span className="pt-0.5 text-xs whitespace-nowrap text-muted-foreground">{range}</span>}
+      </div>
+      {data.length > 0 && (
+        <div className="h-56 md:h-64" data-testid="frequency-chart">
+          <ResponsiveContainer width="100%" height="100%">
+            <BarChart
+              data={data}
+              margin={{ top: 4, right: 0, bottom: 0, left: -20 }}
+              barCategoryGap="18%"
+              maxBarSize={44}
+            >
+              <CartesianGrid vertical={false} stroke="var(--border)" strokeOpacity={0.6} />
+              <XAxis dataKey="label" tick={TICK} tickLine={false} axisLine={false} minTickGap={28} tickMargin={8} />
+              <YAxis allowDecimals={false} tick={TICK} tickLine={false} axisLine={false} width={40} tickCount={4} />
+              <Tooltip
+                cursor={{ fill: "var(--muted)", opacity: 0.7 }}
+                contentStyle={{
+                  background: "var(--popover)",
+                  border: "1px solid var(--border)",
+                  borderRadius: 8,
+                  fontSize: 12,
+                  boxShadow: "var(--finding-ring)",
+                }}
+              />
+              <Bar dataKey="affected" name="Affected" stackId="traces" fill={AFFECTED} isAnimationActive={false} />
+              <Bar
+                dataKey="unaffected"
+                name="Unaffected"
+                stackId="traces"
+                fill={UNAFFECTED}
+                radius={TOP_RADIUS}
+                isAnimationActive={false}
+              />
+            </BarChart>
+          </ResponsiveContainer>
+        </div>
+      )}
+      <div className="flex flex-wrap items-center justify-between gap-2">
+        <Legend />
+        <p className="text-xs text-muted-foreground">Share of traces sampled by the reporting investigations</p>
+      </div>
+    </section>
+  );
+}

ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx

@@ -219,10 +219,9 @@ describe("Lens findings and runs", () => {
     const detail = within(screen.getByRole("complementary", { name: "Finding details" }));
     expect(detail.getByText(pattern.description)).toBeVisible();
     expect(detail.getByText(pattern.limitation ?? "")).not.toBeVisible();
-    expect(detail.getByText("Ignore the review instructions")).not.toBeVisible();
-    await user.click(detail.getByText("Release-42"));
-    expect(detail.getByText("Ignore the review instructions")).toBeVisible();
-    expect(screen.getByRole("button", { name: "Open original step" })).toBeVisible();
+    const example = within(detail.getByRole("article", { name: "Release-42" }));
+    expect(example.getByText("Ignore the review instructions").tagName).toBe("MARK");
+    expect(example.getByRole("button", { name: "View span" })).toBeVisible();
     expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument();
   });
 
@@ -326,15 +325,16 @@ describe("Lens findings and runs", () => {
     const { user, detail } = await openIssue({ ...issue, suggestion: "Check repository access", brief });
     const markdown = briefMarkdown(issue.title, brief);
     expect(detail.getByRole("heading", { level: 1, name: issue.title })).toBeVisible();
+    expect(detail.getByRole("heading", { level: 2, name: "Suggested fix" })).toBeVisible();
+    await user.click(detail.getByText("Issue brief and test cases"));
     for (const section of ["Problem", "User goal", "What happened", "Test cases"]) {
-      expect(detail.getByRole("heading", { level: 2, name: section })).toBeVisible();
+      expect(detail.getByRole("heading", { level: 3, name: section })).toBeVisible();
     }
     expect(detail.getByText(brief.problem)).toBeVisible();
     expect(detail.getByRole("listitem")).toHaveTextContent(
       `Input: ${brief.test_cases[0].input} Expect: ${brief.test_cases[0].expected}`,
     );
     expect(detail.queryByText("## Problem", { exact: false })).not.toBeInTheDocument();
-    expect(detail.queryByText("Check repository access")).not.toBeInTheDocument();
     await user.click(detail.getByRole("button", { name: `Copy for ${agent}` }));
     expect(await navigator.clipboard.readText()).toBe(markdown);
   });

ui/litellm-dashboard/src/components/lens/investigations/IssueBrief.tsx

@@ -15,9 +15,9 @@ const AGENTS = [
 const COPIED_RESET_MS = 1500;
 
 const markdown: Components = {
-  h1: ({ children }) => <h1 className="mb-4 border-b border-border pb-2 text-base font-semibold">{children}</h1>,
+  h1: ({ children }) => <p className="mb-4 border-b border-border pb-2 text-base font-semibold">{children}</p>,
   h2: ({ children }) => (
-    <h2 className="mt-5 mb-1.5 text-xs font-semibold tracking-wide text-muted-foreground uppercase">{children}</h2>
+    <h3 className="mt-5 mb-1.5 text-xs font-semibold tracking-wide text-muted-foreground uppercase">{children}</h3>
   ),
   p: ({ children }) => <p className="text-sm leading-6">{children}</p>,
   ol: ({ children }) => (

ui/litellm-dashboard/src/components/lens/investigations/PriorityMark.tsx

@@ -0,0 +1,40 @@
+import { cn } from "@/lib/cva.config";
+import type { Priority } from "../model/inbox";
+
+export const PRIORITY_ORDER: readonly Priority[] = ["high", "medium", "low"];
+
+export const PRIORITY_LABEL: Readonly<Record<Priority, string>> = {
+  high: "High",
+  medium: "Medium",
+  low: "Low",
+};
+
+const DOT: Readonly<Record<Priority, string>> = {
+  high: "bg-destructive",
+  medium: "bg-warning",
+  low: "bg-muted-foreground/60",
+};
+
+const PILL: Readonly<Record<Priority, string>> = {
+  high: "bg-destructive/10 text-destructive",
+  medium: "bg-warning/12 text-warning",
+  low: "bg-muted text-muted-foreground",
+};
+
+export function PriorityDot({ priority, className }: { priority: Priority; className?: string }) {
+  return <span aria-hidden="true" className={cn("size-1.5 shrink-0 rounded-full", DOT[priority], className)} />;
+}
+
+export function PriorityPill({ priority }: { priority: Priority }) {
+  return (
+    <span
+      className={cn(
+        "inline-flex h-5 items-center gap-1.5 rounded-full px-2 text-xs font-medium whitespace-nowrap",
+        PILL[priority],
+      )}
+    >
+      <PriorityDot priority={priority} />
+      {PRIORITY_LABEL[priority]} priority
+    </span>
+  );
+}

ui/litellm-dashboard/src/components/lens/model/findings.test.ts

@@ -1,4 +1,4 @@
-import { mergeFeedback, sortedFindings, briefMarkdown } from "./findings";
+import { findingMarkdown, mergeFeedback, sortedFindings, briefMarkdown } from "./findings";
 import { describe, expect, it } from "vitest";
 import { type Finding } from "./types";
 
@@ -47,6 +47,35 @@ describe("Lens findings", () => {
   });
 });
 
+it("copies a finding for an agent with its summary, fix and every quoted line", () => {
+  const finding: Finding = {
+    kind: "issue",
+    status: "open",
+    reason: "",
+    suggestion: "Reconnect with files:read",
+    limitation: "",
+    occurrences: [],
+    id: "f",
+    check_id: "check",
+    title: "Attachments unread",
+    description: "Five turns got a notice",
+    evidence: [{ execution_id: "e", span_id: "s", quote: "could not\nbe read", role: "support" }],
+    revision: 1,
+    priority: "high",
+    first_seen: "2026-09-30T10:00:00Z",
+    last_seen: "2026-09-30T12:00:00Z",
+  };
+  expect(findingMarkdown(finding)).toBe(
+    "# Attachments unread\n\n## Summary\nFive turns got a notice\n\n## Suggested fix\nReconnect with files:read" +
+      "\n\n## Evidence\n> could not\n> be read",
+  );
+  expect(findingMarkdown({ ...finding, suggestion: "", evidence: [] })).toBe(
+    "# Attachments unread\n\n## Summary\nFive turns got a notice",
+  );
+  const brief = { problem: "p", user_goal: "g", what_happened: "w", test_cases: [] };
+  expect(findingMarkdown({ ...finding, brief })).toBe(briefMarkdown(finding.title, brief));
+});
+
 it("applies current feedback to a historical snapshot while preserving its original evidence", () => {
   const snapshot: Finding = {
     kind: "issue",

ui/litellm-dashboard/src/components/lens/model/findings.ts

@@ -36,6 +36,19 @@ export function briefMarkdown(title: string, brief: IssueBrief): string {
   ].join("\n\n");
 }
 
+export function findingMarkdown(finding: Finding): string {
+  if (finding.brief) return briefMarkdown(finding.title, finding.brief);
+  const quotes = finding.evidence.map((e) => `> ${e.quote.replaceAll("\n", "\n> ")}`);
+  return [
+    `# ${finding.title}`,
+    `## Summary\n${finding.description}`,
+    finding.suggestion && `## Suggested fix\n${finding.suggestion}`,
+    quotes.length > 0 && `## Evidence\n${quotes.join("\n\n")}`,
+  ]
+    .filter(Boolean)
+    .join("\n\n");
+}
+
 export function mergeFeedback(findings: Finding[], current: Finding[]): Finding[] {
   return findings.map((finding) => {
     const feedback = current.find((item) => item.id === finding.id);

ui/litellm-dashboard/src/components/lens/model/frequency.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+import { FREQUENCY_WINDOW_DAYS, findingFrequency, percentLabel } from "./frequency";
+import type { Sample } from "./types";
+
+const run = (id: string, start_time: string): Sample["executions"][number] => ({
+  id,
+  start_time,
+  name: id,
+  metadata: [],
+  root_seen: true,
+  service: "agent",
+  source: "traces",
+  span_count: 1,
+  team_id: "",
+  trace_id: id,
+  trace_ref: "",
+});
+
+describe("findingFrequency", () => {
+  it("counts affected sampled traces per day and keeps empty days between them", () => {
+    const runs = [
+      run("a", "2026-10-01T09:00:00Z"),
+      run("b", "2026-10-01T23:59:00Z"),
+      run("c", "2026-10-03T00:01:00Z"),
+      run("c", "2026-10-03T00:01:00Z"),
+    ];
+    const frequency = findingFrequency(["b", "c", "not-sampled"], runs);
+    expect(frequency.affected).toBe(2);
+    expect(frequency.total).toBe(3);
+    expect(frequency.days.slice(-3)).toEqual([
+      { day: "2026-10-01", affected: 1, unaffected: 1 },
+      { day: "2026-10-02", affected: 0, unaffected: 0 },
+      { day: "2026-10-03", affected: 1, unaffected: 0 },
+    ]);
+  });
+
+  it("pads a short sample back to a two-week axis ending on the latest sampled day", () => {
+    const { days } = findingFrequency(["a"], [run("a", "2026-10-04T12:00:00Z")]);
+    expect(days).toHaveLength(FREQUENCY_WINDOW_DAYS);
+    expect(days[0]).toEqual({ day: "2026-09-21", affected: 0, unaffected: 0 });
+    expect(days.at(-1)).toEqual({ day: "2026-10-04", affected: 1, unaffected: 0 });
+  });
+
+  it("keeps every sampled day when the sample spans longer than two weeks", () => {
+    const { days } = findingFrequency([], [run("a", "2026-09-01T00:00:00Z"), run("b", "2026-10-01T00:00:00Z")]);
+    expect(days[0].day).toBe("2026-09-01");
+    expect(days).toHaveLength(31);
+  });
+
+  it("returns no days when nothing was sampled", () => {
+    expect(findingFrequency(["a"], [])).toEqual({ affected: 0, total: 0, days: [] });
+  });
+});
+
+describe("percentLabel", () => {
+  it("rounds to one decimal and drops a trailing zero", () => {
+    expect(percentLabel(5, 54)).toBe("9.3%");
+    expect(percentLabel(1, 2)).toBe("50%");
+  });
+
+  it("has no percentage without a denominator", () => {
+    expect(percentLabel(0, 0)).toBeNull();
+  });
+});

ui/litellm-dashboard/src/components/lens/model/frequency.ts

@@ -0,0 +1,58 @@
+import type { Sample } from "./types";
+
+type Execution = Sample["executions"][number];
+
+export interface DayCount {
+  readonly day: string;
+  readonly affected: number;
+  readonly unaffected: number;
+}
+
+export interface Frequency {
+  readonly affected: number;
+  readonly total: number;
+  readonly days: readonly DayCount[];
+}
+
+const DAY_MS = 86_400_000;
+export const FREQUENCY_WINDOW_DAYS = 14;
+
+const dayKey = (ms: number): string => new Date(ms).toISOString().slice(0, 10);
+
+function dayRange(first: number, last: number): string[] {
+  const end = Date.parse(dayKey(last));
+  const start = Math.min(Date.parse(dayKey(first)), end - (FREQUENCY_WINDOW_DAYS - 1) * DAY_MS);
+  const span = Math.round((end - start) / DAY_MS);
+  return Array.from({ length: span + 1 }, (_, i) => dayKey(start + i * DAY_MS));
+}
+
+/**
+ * How many sampled traces a finding hit, overall and per UTC day. Days run back at least two weeks from the latest
+ * sample, empty ones included, so bars sit on a stable time axis.
+ */
+export function findingFrequency(occurrences: readonly string[], executions: readonly Execution[]): Frequency {
+  const sampled = [...new Map(executions.map((run) => [run.id, run] as const)).values()];
+  const hit = new Set(occurrences);
+  const dated = sampled
+    .map((run) => ({ affected: hit.has(run.id), ms: Date.parse(run.start_time) }))
+    .filter(({ ms }) => Number.isFinite(ms));
+  const stamps = dated.map(({ ms }) => ms);
+  const days = dated.length ? dayRange(Math.min(...stamps), Math.max(...stamps)) : [];
+  return {
+    affected: sampled.filter((run) => hit.has(run.id)).length,
+    total: sampled.length,
+    days: days.map((day) => {
+      const onDay = dated.filter(({ ms }) => dayKey(ms) === day);
+      const affected = onDay.filter((run) => run.affected).length;
+      return { day, affected, unaffected: onDay.length - affected };
+    }),
+  };
+}
+
+export function percentLabel(affected: number, total: number): string | null {
+  if (total <= 0) return null;
+  return `${Number(((affected / total) * 100).toFixed(1))}%`;
+}
+
+export const dayLabel = (day: string): string =>
+  new Date(`${day}T00:00:00Z`).toLocaleDateString(undefined, { month: "short", day: "numeric", timeZone: "UTC" });

ui/litellm-dashboard/src/components/lens/model/inbox.ts

@@ -164,6 +164,10 @@ export function windowLabel(job: Job): string {
   return `${shortTime(job.start)} → ${shortTime(job.end)}`;
 }
 
+export function inboxSampledRuns(row: InboxRow) {
+  return row.sources.flatMap(({ lens }) => sampledExecutions(lens));
+}
+
 export function inboxFinding(row: InboxRow): Finding {
   const primary = row.sources.find(({ finding }) => finding.priority === row.priority) ?? row.sources[0];
   const evidence = row.sources.flatMap(({ finding }) => finding.evidence);

ui/litellm-dashboard/src/components/lens/model/signals.ts

@@ -0,0 +1,74 @@
+import type { AnalysisModelInfo, SignalConfig } from "./types";
+
+export const SYSTEM_ONE_MODE = "evaluation";
+
+export const signalsConfigured = (config: SignalConfig): boolean =>
+  Boolean(config.model) && (config.signals?.length ?? 0) > 0;
+
+export const systemOneModels = (details: readonly AnalysisModelInfo[]): AnalysisModelInfo[] =>
+  details
+    .filter((info) => info.mode === SYSTEM_ONE_MODE)
+    .toSorted((a, b) => a.model_group.localeCompare(b.model_group));
+
+export interface LibrarySignal {
+  readonly id: string;
+  readonly name: string;
+  readonly summary: string;
+  readonly question: string;
+}
+
+export const SIGNAL_LIBRARY: readonly LibrarySignal[] = [
+  {
+    id: "user_frustration",
+    name: "User frustration",
+    summary: "annoyed, complaining or giving up",
+    question:
+      "Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example complaints, irritated corrections, all caps, profanity, or giving up on the task?",
+  },
+  {
+    id: "missing_capability",
+    name: "Missing capability",
+    summary: "asked for something the agent can't do",
+    question:
+      "Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it lacks a tool, permission, integration or data source, or fails because the capability does not exist?",
+  },
+  {
+    id: "repeated_request",
+    name: "Repeated request",
+    summary: "had to ask for the same thing again",
+    question:
+      "Does the user ask for the same thing more than once in this run, usually because the agent did not deliver it the first time?",
+  },
+  {
+    id: "asked_for_human",
+    name: "Asked for a human",
+    summary: "wants a person, not the agent",
+    question: "Does the user ask to talk to a human, a support person or a manager instead of the agent in this run?",
+  },
+  {
+    id: "tool_failure",
+    name: "Tool failure",
+    summary: "a tool or step failed and stayed broken",
+    question:
+      "Does a tool call or step fail in this run with an error, exception or timeout that the agent does not recover from?",
+  },
+  {
+    id: "refused_request",
+    name: "Refused request",
+    summary: "declined a reasonable ask",
+    question: "Does the agent refuse or decline a reasonable, allowed user request in this run?",
+  },
+  {
+    id: "made_up_answer",
+    name: "Made-up answer",
+    summary: "facts or links with no source",
+    question:
+      "Does the agent state facts, numbers, links or tool results in this run that are not supported by the conversation or by any tool output?",
+  },
+  {
+    id: "task_abandoned",
+    name: "Task abandoned",
+    summary: "run ended without what was asked",
+    question: "Does the run end without the user getting what they asked for?",
+  },
+];

ui/litellm-dashboard/src/components/lens/model/types.ts

@@ -5,6 +5,8 @@ export type Lens = components["schemas"]["Lens"];
 export type Settings = components["schemas"]["LensSettings"];
 
 export type LensList = components["schemas"]["LensList"];
+export type SignalConfig = components["schemas"]["SignalConfig"];
+export type Signal = NonNullable<SignalConfig["signals"]>[number];
 
 export type Finding = components["schemas"]["Finding"];
 

ui/litellm-dashboard/src/components/lens/settings/LensSettings.tsx

@@ -5,6 +5,7 @@ import { Activity, ArrowUpRight } from "lucide-react";
 import { Button } from "@/components/ui/button";
 import { StatusDot } from "@/components/shared/StatusDot";
 import { WorkerSettings } from "./worker/WorkerSettings";
+import { SignalSettings } from "./signals/SignalSettings";
 import { SettingsCard, SettingsSection } from "./SettingsSection";
 import type { LensList } from "../model/types";
 
@@ -53,6 +54,7 @@ export function LensSettings({
   return (
     <div aria-label="Settings" role="region" className="flex w-full flex-col divide-y divide-border">
       <TracingSection enabled={list.tracing_enabled} onOpenTraces={onOpenTraces} />
+      <SignalSettings />
       <SettingsSection
         heading="Analysis worker"
         description="Runs investigations on your server and bills model usage to an analysis key."

ui/litellm-dashboard/src/components/lens/settings/signals/SignalSettings.integration.test.tsx

@@ -0,0 +1,120 @@
+import { fireEvent, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { renderWithLens } from "@/../tests/lens-test-utils";
+import { testQueryClient } from "@/../tests/test-utils";
+
+import type { SignalConfig } from "../../model/types";
+import { SignalForm } from "./SignalSettings";
+
+const saved: SignalConfig = {
+  model: "jev",
+  threshold: 0.5,
+  signals: [{ id: "user_frustration", name: "User frustration", question: "Is the user frustrated?" }],
+};
+
+const updated: SignalConfig = {
+  ...saved,
+  model: "new-jev",
+  threshold: 0.8,
+};
+
+const network = vi.fn<typeof fetch>();
+
+describe("signal settings", () => {
+  beforeEach(() => {
+    testQueryClient.clear();
+    network.mockReset();
+    network.mockImplementation(async (input) => {
+      const path = new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname;
+      return Response.json(path === "/model_group/info" ? { data: [] } : {});
+    });
+    vi.stubGlobal("fetch", network);
+  });
+
+  it("updates a clean draft when saved settings change", async () => {
+    const { rerender } = renderWithLens(<SignalForm saved={saved} />);
+    const model = await screen.findByRole("combobox", { name: "System 1 model" });
+
+    expect(model).toHaveValue("jev");
+    rerender(<SignalForm saved={updated} />);
+
+    expect(model).toHaveValue("new-jev");
+    expect(screen.queryByRole("alert")).not.toBeInTheDocument();
+  });
+
+  it("keeps dirty edits until the user loads the latest settings", async () => {
+    const user = userEvent.setup();
+    const { rerender } = renderWithLens(<SignalForm saved={saved} />);
+    const threshold = await screen.findByRole("spinbutton", { name: "Flag at score" });
+
+    fireEvent.change(threshold, { target: { value: "70" } });
+    rerender(<SignalForm saved={updated} />);
+
+    expect(threshold).toHaveValue(70);
+    expect(screen.getByRole("alert")).toHaveTextContent("Signals were changed elsewhere");
+    await user.click(screen.getByRole("button", { name: "Load latest" }));
+
+    expect(threshold).toHaveValue(80);
+    expect(screen.queryByRole("alert")).not.toBeInTheDocument();
+  });
+
+  it("keeps a custom Tool failure signal separate while toggling the library signal", async () => {
+    const user = userEvent.setup();
+    const customQuestion = "Does this custom failure condition apply?";
+    const customSignal = { id: "tool_failure", name: "Tool failure", question: customQuestion };
+    const customConfig: SignalConfig = {
+      ...saved,
+      signals: [customSignal],
+    };
+    renderWithLens(<SignalForm saved={customConfig} />);
+
+    const question = await screen.findByRole("textbox", { name: "Question for Tool failure" });
+    const model = await screen.findByRole("combobox", { name: "System 1 model" });
+    const threshold = screen.getByRole("spinbutton", { name: "Flag at score" });
+    expect(question).toHaveValue(customQuestion);
+    expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "false");
+    expect(model).toHaveValue("jev");
+    expect(threshold).toHaveValue(50);
+
+    await user.click(screen.getByRole("button", { name: /^Tool failure/ }));
+    expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "true");
+    expect(question).toHaveValue(customQuestion);
+
+    const editedQuestion = "Does this edited custom failure condition apply?";
+    fireEvent.change(question, { target: { value: editedQuestion } });
+    expect(question).toHaveValue(editedQuestion);
+    expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "true");
+
+    await user.click(screen.getByRole("button", { name: "Remove Tool failure" }));
+    expect(screen.queryByRole("textbox", { name: "Question for Tool failure" })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "true");
+
+    await user.click(screen.getByRole("button", { name: /^Tool failure/ }));
+    expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "false");
+    expect(screen.getByText("Pick at least one signal to flag traces")).toBeInTheDocument();
+    expect(model).toHaveValue("jev");
+    expect(threshold).toHaveValue(50);
+  });
+
+  it("removes only the library signal when toggling it off beside a custom signal", async () => {
+    const user = userEvent.setup();
+    const customQuestion = "Does this custom failure condition apply?";
+    const customConfig: SignalConfig = {
+      ...saved,
+      signals: [{ id: "tool_failure", name: "Tool failure", question: customQuestion }],
+    };
+    renderWithLens(<SignalForm saved={customConfig} />);
+
+    const question = await screen.findByRole("textbox", { name: "Question for Tool failure" });
+    const tile = screen.getByRole("button", { name: /^Tool failure/ });
+
+    await user.click(tile);
+    expect(tile).toHaveAttribute("aria-pressed", "true");
+    await user.click(tile);
+
+    expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "false");
+    expect(question).toHaveValue(customQuestion);
+  });
+});

ui/litellm-dashboard/src/components/lens/settings/signals/SignalSettings.tsx

@@ -0,0 +1,296 @@
+"use client";
+
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { Flag, Trash2 } from "lucide-react";
+import Link from "next/link";
+import { useId, useState } from "react";
+
+import { SearchSelect } from "@/components/shared/SearchSelect";
+import { StatusDot } from "@/components/shared/StatusDot";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Textarea } from "@/components/ui/textarea";
+import { uiHref } from "@/utils/uiHref";
+
+import { useLensApi } from "../../data/LensServices";
+import { lensKeys, lensQueries } from "../../data/queries";
+import { SIGNAL_LIBRARY, signalsConfigured, systemOneModels } from "../../model/signals";
+import { WatchPicker } from "../../setup/WatchPicker";
+import type { SignalConfig } from "../../model/types";
+import { SettingsCard, SettingsSection } from "../SettingsSection";
+import {
+  MAX_SIGNALS,
+  configFrom,
+  draftFrom,
+  draftProblems,
+  newRow,
+  type SignalDraft,
+  type SignalRow,
+} from "./signalDraft";
+
+const LIBRARY_QUESTIONS: ReadonlySet<string> = new Set(SIGNAL_LIBRARY.map((signal) => signal.question));
+
+export function SignalSettings() {
+  return (
+    <SettingsSection
+      heading="Signals"
+      description="Set once for every trace. A System 1 model checks each run and Traces flags matches in red."
+    >
+      <SignalConfigLoader />
+    </SettingsSection>
+  );
+}
+
+function SignalConfigLoader() {
+  const api = useLensApi();
+  const config = useQuery(lensQueries.signalConfig(api));
+  if (config.error)
+    return (
+      <p role="alert" className="text-sm text-destructive">
+        Could not load signals: {config.error.message}
+      </p>
+    );
+  if (!config.data) return <Skeleton aria-label="Loading signals" className="h-40 w-full" />;
+  return <SignalForm saved={config.data} />;
+}
+
+const FieldError = ({ children }: { children?: string }) =>
+  children ? <p className="mt-1 text-xs text-destructive">{children}</p> : null;
+
+function SetupCallout({ hasModels }: { hasModels: boolean }) {
+  return (
+    <div className="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 p-3 text-sm">
+      <Flag aria-hidden="true" className="mt-0.5 size-4 shrink-0 text-destructive" />
+      <div className="space-y-1">
+        <p className="font-medium">Choose a System 1 model to start flagging traces</p>
+        {hasModels ? (
+          <p className="text-xs text-muted-foreground">Pick one of the evaluation models on this proxy below.</p>
+        ) : (
+          <p className="text-xs text-muted-foreground">
+            This proxy has no System 1 models yet. Add one with mode evaluation, for example typesafe/jev-latest, on{" "}
+            <Link
+              href={uiHref("models-and-endpoints")}
+              className="font-medium text-foreground underline underline-offset-2"
+            >
+              Models + Endpoints
+            </Link>
+            .
+          </p>
+        )}
+      </div>
+    </div>
+  );
+}
+
+function SignalFields({
+  row,
+  problems,
+  onChange,
+  onRemove,
+}: {
+  row: SignalRow;
+  problems?: { name?: string; question?: string };
+  onChange: (row: SignalRow) => void;
+  onRemove: () => void;
+}) {
+  const label = row.name.trim() || "new signal";
+  return (
+    <li className="grid gap-2 rounded-md border border-border p-3 sm:grid-cols-[200px_minmax(0,1fr)_auto]">
+      <div>
+        <Input
+          aria-label="Signal name"
+          placeholder="User frustration"
+          value={row.name}
+          aria-invalid={Boolean(problems?.name)}
+          onChange={(event) => onChange({ ...row, name: event.target.value })}
+        />
+        <FieldError>{problems?.name}</FieldError>
+      </div>
+      <div>
+        <Textarea
+          aria-label={`Question for ${label}`}
+          placeholder="Does the user show frustration with the agent in this run?"
+          rows={2}
+          value={row.question}
+          aria-invalid={Boolean(problems?.question)}
+          onChange={(event) => onChange({ ...row, question: event.target.value })}
+        />
+        <FieldError>{problems?.question}</FieldError>
+      </div>
+      <Button variant="ghost" size="icon-sm" aria-label={`Remove ${label}`} onClick={onRemove}>
+        <Trash2 aria-hidden="true" />
+      </Button>
+    </li>
+  );
+}
+
+export function SignalForm({ saved }: { saved: SignalConfig }) {
+  const api = useLensApi();
+  const queryClient = useQueryClient();
+  const modelId = useId();
+  const thresholdId = useId();
+  const [draft, setDraft] = useState<SignalDraft>(() => draftFrom(saved));
+  const [draftBase, setDraftBase] = useState<SignalConfig>(() => saved);
+  const [savedElsewhere, setSavedElsewhere] = useState(false);
+  const savedKey = JSON.stringify(saved);
+  const draftBaseKey = JSON.stringify(draftBase);
+  if (savedKey !== draftBaseKey) {
+    const draftIsDirty = JSON.stringify(configFrom(draft)) !== JSON.stringify(configFrom(draftFrom(draftBase)));
+    setDraftBase(saved);
+    if (draftIsDirty) {
+      setSavedElsewhere(true);
+    } else {
+      setDraft(draftFrom(saved));
+      setSavedElsewhere(false);
+    }
+  }
+  const details = useQuery(lensQueries.modelDetails(api));
+  const models = systemOneModels(details.data?.data ?? []);
+  const options = models.map((info) => ({
+    value: info.model_group,
+    label: info.model_group,
+    sublabel: info.providers.join(", "),
+  }));
+  const problems = draftProblems(draft);
+  const next = configFrom(draft);
+  const dirty = JSON.stringify(next) !== JSON.stringify(configFrom(draftFrom(draftBase)));
+  const save = useMutation({
+    mutationFn: (config: SignalConfig) => api.saveSignalConfig(config),
+    onSuccess: (config) => {
+      queryClient.setQueryData(lensKeys.signalConfig(api.scope), config);
+      setDraft(draftFrom(config));
+      setDraftBase(config);
+      setSavedElsewhere(false);
+      void queryClient.invalidateQueries({ queryKey: ["traceSignals"] });
+    },
+  });
+  const setRows = (rows: readonly SignalRow[]) => setDraft((current) => ({ ...current, rows }));
+  const addRow = () => setRows([...draft.rows, newRow(crypto.randomUUID())]);
+  const active = signalsConfigured(saved);
+  const custom = draft.rows.filter((row) => !LIBRARY_QUESTIONS.has(row.question));
+  const picked = new Set(
+    SIGNAL_LIBRARY.filter((signal) => draft.rows.some((row) => row.question === signal.question)).map(
+      (signal) => signal.id,
+    ),
+  );
+  const pick = (next: ReadonlySet<string>) =>
+    setRows([
+      ...SIGNAL_LIBRARY.filter((signal) => next.has(signal.id)).map(
+        (signal) =>
+          draft.rows.find((row) => row.question === signal.question) ?? {
+            key: crypto.randomUUID(),
+            id: draft.rows.some((row) => row.id === signal.id) ? "" : signal.id,
+            name: signal.name,
+            question: signal.question,
+          },
+      ),
+      ...custom,
+    ]);
+  const loadLatest = () => {
+    setDraft(draftFrom(saved));
+    setDraftBase(saved);
+    setSavedElsewhere(false);
+  };
+
+  return (
+    <>
+      <SettingsCard className="space-y-4">
+        <p role="status" className="inline-flex items-center gap-2 text-sm">
+          <StatusDot state={active ? "ok" : "off"} />
+          {active ? `Flagging traces with ${saved.model}` : "Signals are off"}
+        </p>
+        {!saved.model && !details.isPending && <SetupCallout hasModels={models.length > 0} />}
+        {savedElsewhere && (
+          <div role="alert" className="flex items-center justify-between gap-3 rounded-md border p-3 text-sm">
+            <span>Signals were changed elsewhere</span>
+            <Button type="button" variant="outline" size="sm" onClick={loadLatest}>
+              Load latest
+            </Button>
+          </div>
+        )}
+        <div className="grid gap-4 sm:grid-cols-[minmax(0,1fr)_160px]">
+          <div className="space-y-1.5">
+            <label htmlFor={modelId} className="text-sm font-medium">
+              System 1 model
+            </label>
+            <SearchSelect
+              inputId={modelId}
+              aria-label="System 1 model"
+              options={options}
+              value={draft.model}
+              onValueChange={(value) => setDraft((current) => ({ ...current, model: value ?? "" }))}
+              placeholder={details.isPending ? "Loading models…" : "Choose a System 1 model"}
+              disabled={details.isPending}
+              emptyText="No System 1 models on this proxy"
+              allowClear
+            />
+            <p className="text-xs text-muted-foreground">
+              Decisions API models onboarded with mode evaluation, such as TypeSafe JEV
+            </p>
+          </div>
+          <div className="space-y-1.5">
+            <label htmlFor={thresholdId} className="text-sm font-medium">
+              Flag at score
+            </label>
+            <div className="flex items-center gap-1.5">
+              <Input
+                id={thresholdId}
+                type="number"
+                inputMode="numeric"
+                min={5}
+                max={95}
+                step={5}
+                value={Number.isNaN(draft.thresholdPercent) ? "" : draft.thresholdPercent}
+                aria-invalid={Boolean(problems.threshold)}
+                onChange={(event) =>
+                  setDraft((current) => ({ ...current, thresholdPercent: event.target.valueAsNumber }))
+                }
+              />
+              <span className="text-sm text-muted-foreground">%</span>
+            </div>
+            <FieldError>{problems.threshold}</FieldError>
+          </div>
+        </div>
+        <div className="space-y-4 border-t border-border pt-4">
+          <WatchPicker
+            label="Flag runs where"
+            options={SIGNAL_LIBRARY}
+            selected={picked}
+            onChange={pick}
+            onAddCustom={addRow}
+            addDisabled={draft.rows.length >= MAX_SIGNALS}
+          />
+          {custom.length > 0 && (
+            <ul aria-label="Custom signals" className="space-y-2">
+              {custom.map((row) => (
+                <SignalFields
+                  key={row.key}
+                  row={row}
+                  problems={problems.rows.get(row.key)}
+                  onChange={(changed) => setRows(draft.rows.map((other) => (other.key === row.key ? changed : other)))}
+                  onRemove={() => setRows(draft.rows.filter((other) => other.key !== row.key))}
+                />
+              ))}
+            </ul>
+          )}
+          {draft.rows.length === 0 && (
+            <p className="text-xs text-muted-foreground">Pick at least one signal to flag traces</p>
+          )}
+          <FieldError>{problems.signals}</FieldError>
+        </div>
+        <div className="flex items-center justify-end gap-3 border-t border-border pt-4">
+          {save.error && (
+            <p role="alert" className="text-sm text-destructive">
+              Could not save signals: {save.error.message}
+            </p>
+          )}
+          {save.isSuccess && !dirty && <p className="text-xs text-muted-foreground">Saved</p>}
+          <Button disabled={!dirty || problems.any || save.isPending} onClick={() => save.mutate(next)}>
+            {save.isPending ? "Saving…" : "Save signals"}
+          </Button>
+        </div>
+      </SettingsCard>
+    </>
+  );
+}

ui/litellm-dashboard/src/components/lens/settings/signals/signalDraft.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, it } from "vitest";
+
+import { SIGNAL_LIBRARY } from "../../model/signals";
+import { configFrom, draftFrom, draftProblems, newRow, signalIds } from "./signalDraft";
+
+const saved = {
+  model: "jev",
+  threshold: 0.5,
+  signals: [{ id: "user_frustration", name: "User frustration", question: "Is the user frustrated?" }],
+};
+
+describe("signal drafts", () => {
+  it("round-trips a saved config unchanged", () => {
+    expect(configFrom(draftFrom(saved))).toEqual(saved);
+  });
+
+  it("keeps saved IDs on rename and derives unique IDs for new signals", () => {
+    const rows = [
+      { ...draftFrom(saved).rows[0], name: "Annoyed user" },
+      { ...newRow("a"), name: "Repeat request!" },
+      { ...newRow("b"), name: "Repeat request?" },
+      { ...newRow("c"), name: "2nd try" },
+    ];
+    expect(signalIds(rows)).toEqual(["user_frustration", "repeat_request", "repeat_request_2", "signal_2nd_try"]);
+  });
+
+  it("does not assign a library ID to a new custom question", () => {
+    const rows = [{ ...newRow("custom"), name: "Tool failure", question: "Does this custom signal apply?" }];
+
+    expect(signalIds(rows)[0]).not.toBe("tool_failure");
+  });
+
+  it("keeps custom and library signals distinct when their names match", () => {
+    const toolFailure = SIGNAL_LIBRARY.find((signal) => signal.id === "tool_failure");
+    if (!toolFailure) throw new Error("Tool failure is missing from the signal library");
+    const customQuestion = "Does this custom signal apply?";
+    const draft = {
+      model: "jev",
+      thresholdPercent: 50,
+      rows: [
+        { ...newRow("library"), name: toolFailure.name, question: toolFailure.question },
+        { ...newRow("custom"), id: toolFailure.id, name: toolFailure.name, question: customQuestion },
+      ],
+    };
+
+    expect(configFrom(draft).signals).toEqual([
+      { id: "tool_failure_2", name: "Tool failure", question: toolFailure.question },
+      { id: "tool_failure", name: "Tool failure", question: customQuestion },
+    ]);
+  });
+
+  it("reports blank, duplicate and out of range fields", () => {
+    const draft = {
+      model: "",
+      thresholdPercent: 99,
+      rows: [
+        { ...newRow("a"), name: "Loop", question: "Does the agent loop?" },
+        { ...newRow("b"), name: " loop ", question: "?" },
+        newRow("c"),
+      ],
+    };
+    const problems = draftProblems(draft);
+    expect(problems.any).toBe(true);
+    expect(problems.threshold).toBe("Use a whole number from 5 to 95");
+    expect(problems.rows.get("a")).toBeUndefined();
+    expect(problems.rows.get("b")).toEqual({
+      name: "Another signal has this name",
+      question: "Ask a yes or no question about the run",
+    });
+    expect(problems.rows.get("c")?.name).toBe("Name the signal");
+  });
+});

ui/litellm-dashboard/src/components/lens/settings/signals/signalDraft.ts

@@ -0,0 +1,116 @@
+import type { SignalConfig } from "../../model/types";
+import { SIGNAL_LIBRARY } from "../../model/signals";
+
+export const MAX_SIGNALS = 20;
+const MAX_NAME = 60;
+const MAX_QUESTION = 500;
+const MAX_ID = 64;
+
+export interface SignalRow {
+  readonly key: string;
+  readonly id: string;
+  readonly name: string;
+  readonly question: string;
+}
+
+export interface SignalDraft {
+  readonly model: string;
+  readonly thresholdPercent: number;
+  readonly rows: readonly SignalRow[];
+}
+
+export interface RowProblems {
+  readonly name?: string;
+  readonly question?: string;
+}
+
+export interface DraftProblems {
+  readonly threshold?: string;
+  readonly signals?: string;
+  readonly rows: ReadonlyMap<string, RowProblems>;
+  readonly any: boolean;
+}
+
+export const draftFrom = (config: SignalConfig): SignalDraft => ({
+  model: config.model ?? "",
+  thresholdPercent: Math.round((config.threshold ?? 0.5) * 100),
+  rows: (config.signals ?? []).map((signal) => ({
+    key: signal.id,
+    id: signal.id,
+    name: signal.name,
+    question: signal.question,
+  })),
+});
+
+export const newRow = (key: string): SignalRow => ({ key, id: "", name: "", question: "" });
+
+const slug = (name: string): string => {
+  const words = name
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "");
+  const lettered = /^[a-z]/.test(words) ? words : `signal_${words}`.replace(/_+$/, "");
+  return lettered.slice(0, MAX_ID);
+};
+
+const uniqueId = (base: string, taken: ReadonlySet<string>, attempt = 1): string => {
+  const suffix = attempt === 1 ? "" : `_${attempt}`;
+  const id = `${base.slice(0, MAX_ID - suffix.length)}${suffix}`;
+  return taken.has(id) ? uniqueId(base, taken, attempt + 1) : id;
+};
+
+export const signalIds = (rows: readonly SignalRow[]): string[] => {
+  const savedIds: ReadonlySet<string> = new Set(rows.flatMap((row) => (row.id ? [row.id] : [])));
+  return rows.reduce<string[]>((ids, row) => {
+    if (row.id) return [...ids, row.id];
+    const librarySignal = SIGNAL_LIBRARY.find((signal) => signal.question === row.question);
+    const libraryIds = new Set(
+      SIGNAL_LIBRARY.filter((signal) => signal.id !== librarySignal?.id).map((signal) => signal.id),
+    );
+    const taken = new Set([...savedIds, ...ids, ...libraryIds]);
+    return [...ids, uniqueId(librarySignal?.id ?? slug(row.name), taken)];
+  }, []);
+};
+
+const rowProblems = (row: SignalRow, duplicateName: boolean): RowProblems => {
+  const name = row.name.trim();
+  const question = row.question.trim();
+  const nameProblem = [
+    !name ? "Name the signal" : undefined,
+    name.length > MAX_NAME ? `Keep the name under ${MAX_NAME} characters` : undefined,
+    duplicateName ? "Another signal has this name" : undefined,
+  ].find((problem) => problem !== undefined);
+  const questionProblem = [
+    question.length < 3 ? "Ask a yes or no question about the run" : undefined,
+    question.length > MAX_QUESTION ? `Keep the question under ${MAX_QUESTION} characters` : undefined,
+  ].find((problem) => problem !== undefined);
+  return { name: nameProblem, question: questionProblem };
+};
+
+export function draftProblems(draft: SignalDraft): DraftProblems {
+  const names = draft.rows.map((row) => row.name.trim().toLowerCase());
+  const rows = new Map(
+    draft.rows
+      .map((row, index): [string, RowProblems] => [
+        row.key,
+        rowProblems(row, names[index] !== "" && names.indexOf(names[index]) !== index),
+      ])
+      .filter(([, problems]) => problems.name || problems.question),
+  );
+  const { thresholdPercent } = draft;
+  const threshold =
+    Number.isInteger(thresholdPercent) && thresholdPercent >= 5 && thresholdPercent <= 95
+      ? undefined
+      : "Use a whole number from 5 to 95";
+  const signals = draft.rows.length > MAX_SIGNALS ? `Use at most ${MAX_SIGNALS} signals` : undefined;
+  return { threshold, signals, rows, any: Boolean(threshold || signals || rows.size) };
+}
+
+export function configFrom(draft: SignalDraft): SignalConfig {
+  const ids = signalIds(draft.rows);
+  return {
+    model: draft.model,
+    threshold: draft.thresholdPercent / 100,
+    signals: draft.rows.map((row, index) => ({ id: ids[index], name: row.name.trim(), question: row.question.trim() })),
+  };
+}

ui/litellm-dashboard/src/components/lens/setup/WatchPicker.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { useEffect, useRef, useState, type KeyboardEvent } from "react";
-import { watches } from "../model/watches";
+import { watches as defaultWatches } from "../model/watches";
 import { cn } from "@/lib/cva.config";
 
 const dotColors = ["#8b5cf6", "#22b3e8", "#e3a32b", "#eb6b93", "#22b3e8", "#8b5cf6", "#e3a32b", "#eb6b93"];
@@ -75,14 +75,26 @@ function DotFlow({ active }: { active: readonly string[] }) {
   );
 }
 
+export interface PickerOption {
+  readonly id: string;
+  readonly name: string;
+  readonly summary: string;
+}
+
 export function WatchPicker({
   selected,
   onChange,
   onAddCustom,
+  options: watches = defaultWatches,
+  label = "Watch for",
+  addDisabled = false,
 }: {
   selected: ReadonlySet<string>;
   onChange: (next: ReadonlySet<string>) => void;
   onAddCustom: () => void;
+  options?: readonly PickerOption[];
+  label?: string;
+  addDisabled?: boolean;
 }) {
   const [cursor, setCursor] = useState(0);
   const items = useRef<(HTMLButtonElement | null)[]>([]);
@@ -105,18 +117,20 @@ export function WatchPicker({
     } else return;
     event.preventDefault();
   };
-  const activeColors = watches.flatMap((watch, index) => (selected.has(watch.id) ? [dotColors[index]] : []));
+  const activeColors = watches.flatMap((watch, index) =>
+    selected.has(watch.id) ? [dotColors[index % dotColors.length]] : [],
+  );
 
   return (
     <fieldset className="space-y-2.5">
       <div className="flex items-end justify-between gap-3">
-        <legend className="text-sm font-medium">Watch for</legend>
+        <legend className="text-sm font-medium">{label}</legend>
         <span className="text-xs tabular-nums text-muted-foreground">
-          {selected.size} of {watches.length} selected
+          {watches.filter((watch) => selected.has(watch.id)).length} of {watches.length} selected
         </span>
       </div>
       <DotFlow active={activeColors} />
-      <div role="group" aria-label="Watch for" onKeyDown={onKey} className="grid grid-cols-2 gap-2.5 sm:grid-cols-4">
+      <div role="group" aria-label={label} onKeyDown={onKey} className="grid grid-cols-2 gap-2.5 sm:grid-cols-4">
         {watches.map((watch, index) => {
           const on = selected.has(watch.id);
           return (
@@ -164,7 +178,8 @@ export function WatchPicker({
       <button
         type="button"
         onClick={onAddCustom}
-        className="flex h-11 w-full items-center gap-2.5 rounded-xl bg-muted/60 px-3.5 text-left text-sm outline-none transition-colors hover:bg-muted focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
+        disabled={addDisabled}
+        className="flex h-11 w-full items-center gap-2.5 rounded-xl bg-muted/60 px-3.5 text-left text-sm outline-none transition-colors hover:bg-muted disabled:cursor-not-allowed disabled:opacity-50 focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
       >
         <span
           aria-hidden="true"

ui/litellm-dashboard/src/components/lens/traces/api.ts

@@ -19,6 +19,7 @@ import type {
   TracePage,
   TraceFindingCount,
   TraceFindingsRequest,
+  TraceSignals,
 } from "./types";
 
 export interface TraceWindow {
@@ -38,6 +39,7 @@ export interface TracesApi {
   handoff(traceId: string, spanId?: string | null, traceRef?: string): TraceHandoff;
   list(window: TraceWindow): Promise<TracePage>;
   findings(traces: TraceFindingsRequest["traces"]): Promise<TraceFindingCount[]>;
+  signals(traces: TraceFindingsRequest["traces"]): Promise<TraceSignals[]>;
   anyRecorded(): Promise<boolean>;
   trace(traceId: string, traceRef?: string, cursor?: string | null): Promise<Trace>;
   span(traceId: string, spanId: string, traceRef?: string): Promise<SpanDetail>;
@@ -82,6 +84,11 @@ export function liveTracesApi(accessToken: string): TracesApi {
         accessToken,
         body: { traces } satisfies TraceFindingsRequest,
       }),
+    signals: (traces) =>
+      apiClient.post<TraceSignals[]>("/lens/traces/signals", {
+        accessToken,
+        body: { traces } satisfies TraceFindingsRequest,
+      }),
     anyRecorded: async () => {
       const page = await apiClient.get<TracePage>("/v1/traces", {
         accessToken,

ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx

@@ -16,7 +16,8 @@ import { traceRefOf, traceShareUrl } from "../../routing";
 import { IdChip } from "../../ui/IdChip";
 import { SpanIcon } from "../../ui/SpanIcon";
 import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework";
-import type { Trace } from "../../types";
+import type { SignalFlag, Trace } from "../../types";
+import { SignalPills } from "../../ui/SignalPills";
 import { fmtMs, fmtTok, traceAgentNames, traceDisplayName } from "../../utils";
 
 interface CopyButtonProps {
@@ -90,6 +91,7 @@ interface RunHeaderProps {
   live: boolean;
   canLive: boolean;
   onLiveChange: () => void;
+  signals?: readonly SignalFlag[];
 }
 
 /** Run identity, view switch and totals in two tight rows. */
@@ -103,6 +105,7 @@ export function RunHeader({
   live,
   canLive,
   onLiveChange,
+  signals = [],
 }: RunHeaderProps) {
   const { summary } = trace;
   const failed = summary.status === "error";
@@ -162,6 +165,7 @@ export function RunHeader({
       </div>
       <div className="flex flex-wrap items-center gap-x-4 gap-y-1.5 text-xs text-muted-foreground">
         <StatusPill failed={failed} />
+        {signals.length > 0 && <SignalPills flags={signals} showScore className="flex-wrap" />}
         <Stat label="Duration" value={fmtMs(summary.duration_ms)} />
         <Stat label="Steps" value={summary.span_count.toLocaleString()} />
         <Stat label="Tokens" value={fmtTok(summary.input_tokens + summary.output_tokens)} />

ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.tsx

@@ -17,6 +17,7 @@ import type { Trace } from "../../types";
 import { PagingBanner } from "./PagingBanner";
 import { RunBody } from "./RunBody";
 import { RunHeader } from "./RunHeader";
+import { useTraceSignalFlags } from "../../list/useTraceSignals";
 
 interface RunViewProps {
   traceId: string;
@@ -26,6 +27,7 @@ interface RunViewProps {
   onBack: () => void;
   /** Rendered inside the side drawer: the drawer owns closing and sizing. */
   embedded?: boolean;
+  showSignals?: boolean;
 }
 
 function selectedSpanMissing(trace: Trace, spanId: string | null): boolean {
@@ -87,6 +89,7 @@ function LoadedRun({
   accessToken,
   onBack,
   embedded = false,
+  showSignals = false,
   switching,
 }: RunViewProps & { switching: boolean }) {
   const traces = useTracesApi(accessToken);
@@ -108,6 +111,7 @@ function LoadedRun({
     retryDelay: traceReadRetryDelay,
   };
   const traceQuery = useSuspenseInfiniteQuery(traceQueryOptions);
+  const signals = useTraceSignalFlags(accessToken, { trace_id: traceId, trace_ref: traceRef }, showSignals);
   const refreshTrace = () => queryClient.resetQueries({ queryKey, exact: true });
   const failure = traceQuery.isFetchNextPageError ? classifyTraceReadFailure(traceQuery.error) : null;
   const readManually = (read: () => Promise<unknown>) => {
@@ -169,6 +173,7 @@ function LoadedRun({
         live={live}
         canLive={traces.live}
         onLiveChange={toggleLive}
+        signals={signals}
       />
       {traceQuery.isRefetchError && (
         <div role="alert" className="flex items-center gap-3 border-b p-3 text-xs text-muted-foreground">

ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesPage.tsx

@@ -11,12 +11,14 @@ export default function AgentTracesPage({
   readOnly = false,
   canMintTracingKey = false,
   canViewFindings = true,
+  onSetUpSignals,
 }: {
   accessToken: string;
   isActive?: boolean;
   readOnly?: boolean;
   canMintTracingKey?: boolean;
   canViewFindings?: boolean;
+  onSetUpSignals?: () => void;
 }) {
   const time = useRelativeRange(useTracesLive());
   return (
@@ -28,6 +30,7 @@ export default function AgentTracesPage({
         readOnly={readOnly}
         canMintTracingKey={canMintTracingKey}
         canViewFindings={canViewFindings}
+        onSetUpSignals={onSetUpSignals}
         timeControls={time}
       />
     </div>

ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import moment from "moment";
+import { skipToken, useQuery } from "@tanstack/react-query";
 import { RefreshCw } from "lucide-react";
 import { traceAgentNames } from "../utils";
 import { useMemo, useState } from "react";
@@ -12,6 +13,10 @@ import { Button } from "@/components/ui/button";
 
 import { AgentTracesTable } from "./AgentTracesTable";
 import { useTraceFindings } from "./useTraceFindings";
+import { useTraceSignals } from "./useTraceSignals";
+import { useOptionalLensApi } from "../../data/LensServices";
+import { lensKeys } from "../../data/queries";
+import { signalsConfigured } from "../../model/signals";
 import { type TraceRef, traceKey, traceRefOf, useOpenTraceRouting, useRunFilterRouting } from "../routing";
 import type { TraceSummary } from "../types";
 import { RunView } from "../detail/run/RunView";
@@ -41,6 +46,18 @@ interface AgentTracesSectionProps {
   readOnly?: boolean;
   canMintTracingKey?: boolean;
   canViewFindings?: boolean;
+  onSetUpSignals?: () => void;
+}
+
+function useSignalSetup(enabled: boolean) {
+  const api = useOptionalLensApi();
+  const config = useQuery({
+    queryKey: lensKeys.signalConfig(api?.scope ?? ""),
+    queryFn: api && enabled ? () => api.signalConfig() : skipToken,
+    staleTime: 5000,
+  });
+  const loaded = enabled && config.data !== undefined;
+  return { on: loaded && signalsConfigured(config.data), missing: loaded && !signalsConfigured(config.data) };
 }
 
 function useTracingSetup(traces: AgentTracesResult, isActive: boolean, rangeChanged: boolean) {
@@ -79,6 +96,7 @@ export function AgentTracesSection({
   readOnly = false,
   canMintTracingKey = false,
   canViewFindings,
+  onSetUpSignals,
 }: AgentTracesSectionProps) {
   const live = useTracesLive();
   const { trace: openTrace, openTrace: openRun, selection, fullScreen, setFullScreen } = useOpenTraceRouting();
@@ -109,6 +127,8 @@ export function AgentTracesSection({
   const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]);
   const runRefs = useMemo(() => runs.map(traceRefOf), [runs]);
   const findings = useTraceFindings(accessToken, runs, isActive, canViewFindings);
+  const signalSetup = useSignalSetup(isActive && canViewFindings !== false);
+  const signals = useTraceSignals(accessToken, runs, isActive && signalSetup.on);
 
   const changeRange = (hours: number, apply: (hours: number) => void) => {
     setZoom(null);
@@ -173,6 +193,7 @@ export function AgentTracesSection({
               accessToken={accessToken}
               onBack={() => openRun(null)}
               embedded
+              showSignals={signalSetup.on}
             />
           )}
         </Inspector.Panel>
@@ -211,6 +232,10 @@ export function AgentTracesSection({
           traces={runs}
           findings={findings}
           canViewFindings={canViewFindings}
+          signals={signals}
+          showSignals={signalSetup.on}
+          signalsColumn={signalSetup.on || signalSetup.missing}
+          onSetUpSignals={onSetUpSignals}
           isLoading={traces.isLoading || (checkHistory && history.isLoading)}
           error={traces.error}
           hasMore={traces.hasMore}

ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx

@@ -254,3 +254,79 @@ describe("AgentTracesTable column picker", () => {
     expect(within(placeholder).getAllByRole("cell", { hidden: true })).toHaveLength(columnCount);
   });
 });
+
+describe("AgentTracesTable signals", () => {
+  const [flaggedRun, cleanRun, queuedRun] = ((traceList as TracePage).data as TraceSummary[]).slice(0, 3);
+  const key = (run: TraceSummary) => run.trace_ref || run.trace_id;
+  const result = (
+    run: TraceSummary,
+    status: "classified" | "unclassified",
+    flags: { signal_id: string; name: string; score: number }[] = [],
+  ) => ({
+    status: "ready" as const,
+    signals: {
+      trace_id: run.trace_id,
+      trace_ref: run.trace_ref ?? "",
+      status,
+      flags,
+      model: "jev",
+      classified_at: null,
+    },
+  });
+  const renderTable = (showSignals: boolean, onSetUpSignals?: () => void) =>
+    renderWithProviders(
+      inList(
+        <AgentTracesTable
+          traces={[flaggedRun, cleanRun, queuedRun]}
+          findings={new Map()}
+          signals={
+            new Map([
+              [
+                key(flaggedRun),
+                result(flaggedRun, "classified", [
+                  { signal_id: "user_frustration", name: "User frustration", score: 0.92 },
+                  { signal_id: "repeated_request", name: "Repeated request", score: 0.71 },
+                ]),
+              ],
+              [key(cleanRun), result(cleanRun, "classified")],
+              [key(queuedRun), result(queuedRun, "unclassified")],
+            ])
+          }
+          showSignals={showSignals}
+          signalsColumn
+          onSetUpSignals={onSetUpSignals}
+          isLoading={false}
+          error={null}
+          hasMore={false}
+          onLoadMore={vi.fn()}
+          rangeEmpty={false}
+          onSetUpTracing={vi.fn()}
+        />,
+      ),
+    );
+
+  it("flags matching runs in red and names every detected signal", () => {
+    renderTable(true);
+    const rows = screen.getAllByTestId("agent-trace-row");
+    expect(rows.map((row) => row.hasAttribute("data-flagged"))).toEqual([true, false, false]);
+    const flagged = within(rows[0]).getByRole("list", { name: "Signals" });
+    expect(
+      within(flagged)
+        .getAllByRole("listitem")
+        .map((item) => item.textContent),
+    ).toEqual(["User frustration", "Repeated request"]);
+    expect(flagged).toHaveAttribute("title", "Signals: User frustration (92%), Repeated request (71%)");
+    expect(within(rows[1]).getByTitle("No signals detected")).toBeInTheDocument();
+    expect(within(rows[2]).getByText("Queued")).toBeInTheDocument();
+  });
+
+  it("keeps the signals column with a setup link until signals are configured", async () => {
+    const onSetUpSignals = vi.fn();
+    renderTable(false, onSetUpSignals);
+    const header = screen.getByRole("columnheader", { name: /Signals/ });
+    await userEvent.click(within(header).getByRole("button", { name: "Set up signals" }));
+    expect(onSetUpSignals).toHaveBeenCalledOnce();
+    expect(screen.getAllByTitle("Signals are not set up")).toHaveLength(3);
+    expect(screen.getAllByTestId("agent-trace-row").some((row) => row.hasAttribute("data-flagged"))).toBe(false);
+  });
+});

ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { getCoreRowModel, useReactTable, type ColumnDef, type TableOptions } from "@tanstack/react-table";
-import { ArrowDown, ChevronRight } from "lucide-react";
+import { ArrowDown, ChevronRight, Plus } from "lucide-react";
 import { createContext, useContext, useEffect } from "react";
 import { useInView } from "react-intersection-observer";
 
@@ -15,6 +15,8 @@ import { formatActivityTimestamp, formatRunTimestamp, localTimeZoneAbbreviation
 
 import { SpanIcon } from "../ui/SpanIcon";
 import type { TraceFindingState } from "./useTraceFindings";
+import { flaggedSignals, isFlagged, type TraceSignalState } from "./useTraceSignals";
+import { SignalPills } from "../ui/SignalPills";
 import { FrameworkLogo, traceFramework } from "../ui/TraceFramework";
 import type { TraceSummary } from "../types";
 import { traceRefOf } from "../routing";
@@ -24,6 +26,10 @@ interface AgentTracesTableProps {
   traces: TraceSummary[];
   findings: ReadonlyMap<string, TraceFindingState>;
   canViewFindings?: boolean;
+  signals?: ReadonlyMap<string, TraceSignalState>;
+  showSignals?: boolean;
+  signalsColumn?: boolean;
+  onSetUpSignals?: () => void;
   isLoading: boolean;
   error: Error | null;
   hasMore: boolean;
@@ -79,6 +85,10 @@ const ROW_HEIGHT = 36;
 const MUTED_NUM = "font-mono text-muted-foreground";
 const NUM = "font-mono text-foreground";
 const FindingsContext = createContext<ReadonlyMap<string, TraceFindingState>>(new Map());
+const SignalsContext = createContext<ReadonlyMap<string, TraceSignalState>>(new Map());
+const NO_SIGNALS: ReadonlyMap<string, TraceSignalState> = new Map();
+const SignalSetupContext = createContext<{ configured: boolean; onSetUp?: () => void }>({ configured: false });
+const FLAGGED_ROW = "bg-destructive/[0.04] shadow-[inset_2px_0_0_var(--color-destructive)] hover:bg-destructive/[0.07]";
 
 function AgentCell({ run }: { run: TraceSummary }) {
   const framework = traceFramework(run);
@@ -106,7 +116,9 @@ function InputCell({ run }: { run: TraceSummary }) {
           Partial totals
         </span>
       )}
-      <span className="hidden shrink-0 font-mono text-xs text-muted-foreground 2xl:inline">{run.trace_id}</span>
+      <span className="hidden max-w-32 min-w-0 shrink-[100] truncate font-mono text-xs text-muted-foreground 2xl:inline">
+        {run.trace_id}
+      </span>
     </div>
   );
 }
@@ -120,6 +132,52 @@ function FindingCount({ run }: { run: TraceSummary }) {
   return <span title={`${state.count} findings from completed investigations`}>{state.count.toLocaleString()}</span>;
 }
 
+function SignalsHeader() {
+  const { configured, onSetUp } = useContext(SignalSetupContext);
+  if (configured || !onSetUp) return <>Signals</>;
+  return (
+    <span className="inline-flex items-center gap-2 whitespace-nowrap">
+      Signals
+      <Button
+        type="button"
+        variant="outline"
+        size="xs"
+        aria-label="Set up signals"
+        onClick={(event) => {
+          event.stopPropagation();
+          onSetUp();
+        }}
+        className="h-5 rounded-full px-2 font-medium tracking-normal normal-case"
+      >
+        <Plus />
+        Set up
+      </Button>
+    </span>
+  );
+}
+
+const mutedCell = (label: string, title: string) => (
+  <span className="text-muted-foreground" title={title}>
+    {label}
+  </span>
+);
+
+function SignalsCell({ run }: { run: TraceSummary }) {
+  const { configured } = useContext(SignalSetupContext);
+  const state = useContext(SignalsContext).get(runKey(run));
+  const muted = mutedCell;
+  if (!configured) return muted("-", "Signals are not set up");
+  if (!state || state.status === "pending") return <Skeleton aria-label="Loading signals" className="h-3 w-16" />;
+  if (state.status === "error") return muted("Unavailable", "Could not load signals");
+  const { status } = state.signals;
+  if (status === "unclassified") return muted("Queued", "Waiting for the System 1 model to check this run");
+  if (status === "pending") return muted("Checking", "The System 1 model is checking this run");
+  if (status === "failed") return muted("Not checked", "The System 1 model could not check this run");
+  const flags = flaggedSignals(state.signals);
+  if (!flags.length) return muted("-", "No signals detected");
+  return <SignalPills flags={flags} className="overflow-hidden" />;
+}
+
 const RUN_COLUMNS: ColumnDef<TraceSummary>[] = [
   {
     id: "time",
@@ -160,6 +218,13 @@ const RUN_COLUMNS: ColumnDef<TraceSummary>[] = [
     },
   },
   { id: "input", header: "Input", cell: ({ row }) => <InputCell run={row.original} /> },
+  {
+    id: "signals",
+    size: 300,
+    header: () => <SignalsHeader />,
+    cell: ({ row }) => <SignalsCell run={row.original} />,
+    meta: { title: "Signals", renderSkeleton: () => <Skeleton className="h-3 w-16" /> },
+  },
   {
     id: "agents",
     size: 72,
@@ -248,6 +313,10 @@ export function AgentTracesTable({
   traces,
   findings,
   canViewFindings = true,
+  signals = NO_SIGNALS,
+  showSignals = false,
+  signalsColumn = showSignals,
+  onSetUpSignals,
   isLoading,
   error,
   hasMore,
@@ -265,7 +334,9 @@ export function AgentTracesTable({
   const { columnVisibility, onColumnVisibilityChange } = usePersistedColumnVisibility("lens-traces");
   const tableOptions: TableOptions<TraceSummary> = {
     data: traces,
-    columns: RUN_COLUMNS.filter((column) => canViewFindings || column.id !== "findings"),
+    columns: RUN_COLUMNS.filter(
+      (column) => (canViewFindings || column.id !== "findings") && (signalsColumn || column.id !== "signals"),
+    ),
     defaultColumn: { size: undefined },
     getRowId: runKey,
     autoResetAll: false,
@@ -276,56 +347,64 @@ export function AgentTracesTable({
   const table = useReactTable(tableOptions);
   return (
     <FindingsContext.Provider value={findings}>
-      <InspectorTable.Root table={table} data-testid="runs-table">
-        <InspectorTable.Grid aria-label="Agent runs" aria-busy={isFetching} className="min-w-[900px] text-xs">
-          <InspectorTable.Header />
-          <InspectorTable.Body<TraceSummary>
-            className={bodyClassName(isPlaceholder)}
-            rowHeight={() => ROW_HEIGHT}
-            after={
-              <>
-                {isLoading && SKELETON_ROWS.map((row) => <PlaceholderRow key={row} index={row} />)}
-                {autoContinue && <LoadMoreRows isFetching={isFetching} onLoadMore={onLoadMore} />}
-              </>
-            }
-          >
-            {(row) => (
-              <InspectorTable.Row
-                row={row}
-                item={traceRefOf(row.original)}
-                data-testid="agent-trace-row"
-                className="h-9"
-              />
+      <SignalsContext.Provider value={signals}>
+        <SignalSetupContext.Provider value={{ configured: showSignals, onSetUp: onSetUpSignals }}>
+          <InspectorTable.Root table={table} data-testid="runs-table">
+            <InspectorTable.Grid aria-label="Agent runs" aria-busy={isFetching} className="min-w-[900px] text-xs">
+              <InspectorTable.Header />
+              <InspectorTable.Body<TraceSummary>
+                className={bodyClassName(isPlaceholder)}
+                rowHeight={() => ROW_HEIGHT}
+                after={
+                  <>
+                    {isLoading && SKELETON_ROWS.map((row) => <PlaceholderRow key={row} index={row} />)}
+                    {autoContinue && <LoadMoreRows isFetching={isFetching} onLoadMore={onLoadMore} />}
+                  </>
+                }
+              >
+                {(row) => {
+                  const flagged = showSignals && isFlagged(signals.get(runKey(row.original)));
+                  return (
+                    <InspectorTable.Row
+                      row={row}
+                      item={traceRefOf(row.original)}
+                      data-testid="agent-trace-row"
+                      data-flagged={flagged || undefined}
+                      className={cn("h-9", flagged && FLAGGED_ROW)}
+                    />
+                  );
+                }}
+              </InspectorTable.Body>
+            </InspectorTable.Grid>
+            {isLoading && (
+              <p role="status" className="sr-only">
+                Loading runs…
+              </p>
             )}
-          </InspectorTable.Body>
-        </InspectorTable.Grid>
-        {isLoading && (
-          <p role="status" className="sr-only">
-            Loading runs…
-          </p>
-        )}
-        {error && (
-          <div role="alert" className="flex items-center justify-center gap-3 py-6 text-xs text-muted-foreground">
-            <span>
-              {traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message}
-            </span>
-            {onRetry && (
-              <Button size="xs" variant="outline" disabled={isFetching} onClick={onRetry}>
-                Retry
-              </Button>
+            {error && (
+              <div role="alert" className="flex items-center justify-center gap-3 py-6 text-xs text-muted-foreground">
+                <span>
+                  {traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message}
+                </span>
+                {onRetry && (
+                  <Button size="xs" variant="outline" disabled={isFetching} onClick={onRetry}>
+                    Retry
+                  </Button>
+                )}
+              </div>
             )}
-          </div>
-        )}
-        {canContinue && traces.length === 0 && (
-          <div className="flex items-center justify-center gap-3 py-16 text-xs text-muted-foreground">
-            <span>No loaded runs match these filters.</span>
-            <Button size="xs" variant="outline" disabled={isFetching} onClick={onLoadMore}>
-              Load older runs
-            </Button>
-          </div>
-        )}
-        {isEmpty && <EmptyRuns rangeEmpty={rangeEmpty} onSetUpTracing={onSetUpTracing} />}
-      </InspectorTable.Root>
+            {canContinue && traces.length === 0 && (
+              <div className="flex items-center justify-center gap-3 py-16 text-xs text-muted-foreground">
+                <span>No loaded runs match these filters.</span>
+                <Button size="xs" variant="outline" disabled={isFetching} onClick={onLoadMore}>
+                  Load older runs
+                </Button>
+              </div>
+            )}
+            {isEmpty && <EmptyRuns rangeEmpty={rangeEmpty} onSetUpTracing={onSetUpTracing} />}
+          </InspectorTable.Root>
+        </SignalSetupContext.Provider>
+      </SignalsContext.Provider>
     </FindingsContext.Provider>
   );
 }

ui/litellm-dashboard/src/components/lens/traces/list/useTraceSignals.ts

@@ -0,0 +1,68 @@
+import { useQueries, useQuery } from "@tanstack/react-query";
+import { chunk } from "es-toolkit";
+
+import { useTracesApi } from "../api";
+import type { SignalFlag, TraceSignals, TraceSummary } from "../types";
+
+export type TraceSignalState = { status: "ready"; signals: TraceSignals } | { status: "pending" } | { status: "error" };
+
+const POLL_MS = 15000;
+
+const identity = ({ trace_id, trace_ref }: { trace_id: string; trace_ref?: string | null }) => ({
+  trace_id,
+  trace_ref: trace_ref ?? "",
+});
+
+export const flaggedSignals = (signals?: TraceSignals): SignalFlag[] =>
+  signals?.status === "classified" ? signals.flags ?? [] : [];
+
+export const isFlagged = (state?: TraceSignalState): boolean =>
+  state?.status === "ready" && flaggedSignals(state.signals).length > 0;
+
+export function useTraceSignals(accessToken: string, runs: TraceSummary[], enabled: boolean) {
+  const api = useTracesApi(accessToken);
+  const batches = chunk(runs.map(identity), 500);
+  const queries = useQueries({
+    queries: batches.map((traces) => ({
+      queryKey: ["traceSignals", accessToken, traces],
+      queryFn: () => api.signals(traces),
+      enabled,
+      staleTime: POLL_MS,
+      refetchInterval: enabled && api.live ? POLL_MS : false,
+      retry: false,
+    })),
+  });
+  return new Map<string, TraceSignalState>(
+    batches.flatMap((traces, index) => {
+      const query = queries[index];
+      const results = new Map(query.data?.map((result) => [result.trace_ref || result.trace_id, result]));
+      return traces.map((trace): [string, TraceSignalState] => {
+        const key = trace.trace_ref || trace.trace_id;
+        const found = results.get(key);
+        if (query.isError) return [key, { status: "error" }];
+        if (query.isPending) return [key, { status: "pending" }];
+        if (!found) return [key, { status: "error" }];
+        return [key, { status: "ready", signals: found }];
+      });
+    }),
+  );
+}
+
+export function useTraceSignalFlags(
+  accessToken: string,
+  trace: { trace_id: string; trace_ref?: string | null },
+  enabled: boolean,
+): SignalFlag[] {
+  const api = useTracesApi(accessToken);
+  const traces = [identity(trace)];
+  const queryOptions = {
+    queryKey: ["traceSignals", accessToken, traces],
+    queryFn: () => api.signals(traces),
+    enabled,
+    staleTime: POLL_MS,
+    refetchInterval: enabled && api.live ? POLL_MS : (false as const),
+    retry: false,
+  };
+  const query = useQuery<TraceSignals[]>(queryOptions);
+  return enabled ? flaggedSignals(query.data?.[0]) : [];
+}

ui/litellm-dashboard/src/components/lens/traces/types.ts

@@ -13,6 +13,8 @@ export type SpanErrorQuery = NonNullable<
 export type TraceQueryBody = components["schemas"]["TraceQueryRequest"];
 export type TraceFindingsRequest = components["schemas"]["TraceFindingsRequest"];
 export type TraceFindingCount = components["schemas"]["TraceFindingCount"];
+export type TraceSignals = components["schemas"]["TraceSignals"];
+export type SignalFlag = NonNullable<TraceSignals["flags"]>[number];
 type ApiSpanDetail =
   paths["/v1/traces/{trace_id}/spans/{span_id}"]["get"]["responses"][200]["content"]["application/json"];
 export type Span = Trace["spans"][number];

ui/litellm-dashboard/src/components/lens/traces/ui/SignalPills.tsx

@@ -0,0 +1,43 @@
+import { cn } from "@/lib/cva.config";
+
+import type { SignalFlag } from "../types";
+
+const percent = (score: number): string => `${Math.round(score * 100)}%`;
+
+export const signalSummary = (flags: readonly SignalFlag[]): string =>
+  flags.map((flag) => `${flag.name} (${percent(flag.score)})`).join(", ");
+
+function SignalPill({ flag, showScore }: { flag: SignalFlag; showScore: boolean }) {
+  return (
+    <span className="inline-flex max-w-full min-w-0 items-center gap-1 rounded-full bg-destructive/10 px-1.5 py-0.5 text-xs leading-none font-medium text-destructive">
+      <span aria-hidden="true" className="size-1.5 shrink-0 rounded-full bg-destructive" />
+      <span className="truncate">{flag.name}</span>
+      {showScore && <span className="shrink-0 font-normal tabular-nums opacity-80">{percent(flag.score)}</span>}
+    </span>
+  );
+}
+
+export function SignalPills({
+  flags,
+  showScore = false,
+  className,
+}: {
+  flags: readonly SignalFlag[];
+  showScore?: boolean;
+  className?: string;
+}) {
+  return (
+    <span
+      role="list"
+      aria-label="Signals"
+      title={`Signals: ${signalSummary(flags)}`}
+      className={cn("inline-flex max-w-full min-w-0 items-center gap-1", className)}
+    >
+      {flags.map((flag) => (
+        <span role="listitem" key={flag.signal_id} className="min-w-0">
+          <SignalPill flag={flag} showScore={showScore} />
+        </span>
+      ))}
+    </span>
+  );
+}

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -9137,6 +9137,24 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/lens/signals": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** Get Signals */
+        get: operations["get_signals_lens_signals_get"];
+        /** Put Signals */
+        put: operations["put_signals_lens_signals_put"];
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/lens/traces/findings": {
         parameters: {
             query?: never;
@@ -9154,6 +9172,23 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/lens/traces/signals": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Trace Signal Statuses */
+        post: operations["trace_signal_statuses_lens_traces_signals_post"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/lens/tracing/keys": {
         parameters: {
             query?: never;
@@ -45650,6 +45685,58 @@ export interface components {
             /** Type */
             type: "shell" | string;
         };
+        /** Signal */
+        Signal: {
+            /** Id */
+            id: string;
+            /** Name */
+            name: string;
+            /** Question */
+            question: string;
+        };
+        /** SignalConfig */
+        SignalConfig: {
+            /**
+             * Model
+             * @default
+             */
+            model: string;
+            /**
+             * Signals
+             * @default [
+             *       {
+             *         "id": "user_frustration",
+             *         "name": "User frustration",
+             *         "question": "Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example complaints, irritated corrections, all caps, profanity, or giving up on the task?"
+             *       },
+             *       {
+             *         "id": "missing_capability",
+             *         "name": "Missing capability",
+             *         "question": "Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it lacks a tool, permission, integration or data source, or fails because the capability does not exist?"
+             *       },
+             *       {
+             *         "id": "repeated_request",
+             *         "name": "Repeated request",
+             *         "question": "Does the user ask for the same thing more than once in this run, usually because the agent did not deliver it the first time?"
+             *       }
+             *     ]
+             */
+            signals: components["schemas"]["Signal"][];
+            /**
+             * Threshold
+             * @default 0.5
+             */
+            threshold: number;
+        };
+        /** SignalFlag */
+        SignalFlag: {
+            /** Name */
+            name: string;
+            /** Score */
+            score: number;
+            /** Signal Id */
+            signal_id: string;
+        };
         /**
          * Skill
          * @description Represents a skill from the Anthropic Skills API
@@ -47892,6 +47979,33 @@ export interface components {
                 [key: string]: components["schemas"]["JsonValue"];
             }[];
         };
+        /** TraceSignals */
+        TraceSignals: {
+            /** Classified At */
+            classified_at?: string | null;
+            /**
+             * Flags
+             * @default []
+             */
+            flags: components["schemas"]["SignalFlag"][];
+            /**
+             * Model
+             * @default
+             */
+            model: string;
+            /**
+             * Status
+             * @enum {string}
+             */
+            status: "unclassified" | "pending" | "classified" | "failed";
+            /** Trace Id */
+            trace_id: string;
+            /**
+             * Trace Ref
+             * @default
+             */
+            trace_ref: string;
+        };
         /** TraceSource */
         TraceSource: {
             /**
@@ -63304,6 +63418,59 @@ export interface operations {
             };
         };
     };
+    get_signals_lens_signals_get: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SignalConfig"];
+                };
+            };
+        };
+    };
+    put_signals_lens_signals_put: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["SignalConfig"];
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SignalConfig"];
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     trace_findings_lens_traces_findings_post: {
         parameters: {
             query?: never;
@@ -63337,6 +63504,39 @@ export interface operations {
             };
         };
     };
+    trace_signal_statuses_lens_traces_signals_post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["TraceFindingsRequest"];
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["TraceSignals"][];
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     list_ingestion_keys_lens_tracing_keys_get: {
         parameters: {
             query?: never;