chore: merge main into litellm_remove_lit002_dict_ban

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/codeql/codeql-config.yml

@@ -1,8 +1,5 @@
 name: "LiteLLM CodeQL config"
 
-queries:
-  - uses: security-and-quality
-
 # Known OOM queries on large Python codebases:
 # CodeQL builds a full data flow graph in memory. These two queries trace
 # sensitive data through every log call / regex pattern, causing combinatorial
@@ -14,17 +11,6 @@ query-filters:
       id: py/clear-text-logging-sensitive-data  # CWE-312
   - exclude:
       id: py/polynomial-redos                   # CWE-730
-  # Import resolution confuses stdlib types with management_endpoints/types.py.
-  # The generic cycle query also reports intentional deferred imports.
-  - exclude:
-      id: py/cyclic-import
-  - exclude:
-      id: py/unsafe-cyclic-import
-  # Known false positives on live settings and Protocol placeholders.
-  - exclude:
-      id: py/unused-global-variable
-  - exclude:
-      id: py/ineffectual-statement
 
 paths-ignore:
   - tests

.github/workflows/codeql.yml

@@ -1,8 +1,6 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main]
   schedule:
@@ -43,14 +41,15 @@ jobs:
           persist-credentials: false
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
+        uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           languages: ${{ matrix.language }}
           build-mode: ${{ matrix.build-mode }}
           config-file: ./.github/codeql/codeql-config.yml
+          queries: ${{ github.event_name == 'pull_request' && '+security-extended' || '' }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
+        uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           category: "/language:${{ matrix.language }}"
           output: sarif-results
@@ -83,7 +82,7 @@ jobs:
           output: sarif-results/python.sarif
 
       - name: Upload SARIF
-        uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
+        uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           sarif_file: sarif-results
           category: "/language:${{ matrix.language }}"

helm/litellm-helm/templates/migrations-job.yaml

@@ -6,6 +6,9 @@ metadata:
   name: {{ include "litellm.fullname" . }}-migrations
   labels:
     {{- include "litellm.labels" . | nindent 4 }}
+    {{- with .Values.migrationJob.jobLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   annotations:
     {{- if .Values.migrationJob.hooks.argocd.enabled }}
     argocd.argoproj.io/hook: PreSync
@@ -17,6 +20,9 @@ metadata:
     helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }}
     {{- end }}
     checksum/config: {{ toYaml .Values | sha256sum }}
+    {{- with .Values.migrationJob.jobAnnotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
   template:
     metadata:
@@ -25,6 +31,9 @@ spec:
         {{- with .Values.podLabels }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
+        {{- with .Values.migrationJob.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       annotations:
         {{- with .Values.migrationJob.annotations }}
         {{- toYaml . | nindent 8 }}
@@ -47,7 +56,16 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
+          {{- if .Values.migrationJob.command }}
+          command: {{ toYaml .Values.migrationJob.command | nindent 12 }}
+          {{- else }}
           command: ["python", "litellm/proxy/prisma_migration.py"]
+          {{- end }}
+          {{- if .Values.migrationJob.args }}
+          args: {{ toYaml .Values.migrationJob.args | nindent 12 }}
+          {{- else if .Values.migrationJob.command }}
+          args: []
+          {{- end }}
           workingDir: "/app"
           env:
             {{- if .Values.db.useExisting }}

helm/litellm-helm/tests/migrations-job_tests.yaml

@@ -360,3 +360,73 @@ tests:
     asserts:
       - notExists:
           path: spec.activeDeadlineSeconds
+
+  - it: should set custom jobLabels and jobAnnotations on Job metadata
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        jobLabels:
+          environment: production
+          team: platform
+        jobAnnotations:
+          example.com/cost-center: "1234"
+    asserts:
+      - equal:
+          path: metadata.labels.environment
+          value: production
+      - equal:
+          path: metadata.labels.team
+          value: platform
+      - equal:
+          path: metadata.annotations['example.com/cost-center']
+          value: "1234"
+
+  - it: should set custom podLabels on Pod template
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        podLabels:
+          custom.io/pod-role: migration
+    asserts:
+      - equal:
+          path: spec.template.metadata.labels['custom.io/pod-role']
+          value: migration
+
+  - it: should override container command and args
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        command:
+          - sh
+        args:
+          - -c
+          - echo migrating
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].command
+          value:
+            - sh
+      - equal:
+          path: spec.template.spec.containers[0].args
+          value:
+            - -c
+            - echo migrating
+
+  - it: should clear container args when only command is specified
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        command:
+          - sh
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].command
+          value:
+            - sh
+      - equal:
+          path: spec.template.spec.containers[0].args
+          value: []

helm/litellm-helm/values.yaml

@@ -572,6 +572,11 @@ migrationJob:
   # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
   serviceAccountName: ""
   annotations: {}
+  jobLabels: {} # Custom labels for the Job metadata
+  jobAnnotations: {} # Custom annotations for the Job metadata
+  podLabels: {} # Custom labels for the Job pod template
+  command: [] # Override container command (defaults to ["python", "litellm/proxy/prisma_migration.py"])
+  args: [] # Override container args
   ttlSecondsAfterFinished: 120
   resources: {}
   #  Unset by default. This job runs the database migration and exits, so it does not

litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000000_lens_due_at/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_Lens" ADD COLUMN IF NOT EXISTS "due_at" TIMESTAMP(3) DEFAULT '1970-01-01 00:00:00';

litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000100_lens_due_at_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_due_at_idx"
+ON "LiteLLM_Lens" ("due_at");

litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000200_lens_trace_signals/migration.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_LensSignalConfig" (
+    "id" TEXT NOT NULL,
+    "data" JSONB NOT NULL,
+    CONSTRAINT "LiteLLM_LensSignalConfig_pkey" PRIMARY KEY ("id")
+);
+
+CREATE TABLE IF NOT EXISTS "LiteLLM_LensTraceSignal" (
+    "trace_id" TEXT NOT NULL,
+    "trace_ref" TEXT NOT NULL DEFAULT '',
+    "config_key" TEXT NOT NULL,
+    "span_count" INTEGER NOT NULL,
+    "claimed_until" TIMESTAMP(3),
+    "classified_at" TIMESTAMP(3),
+    "data" JSONB NOT NULL,
+    CONSTRAINT "LiteLLM_LensTraceSignal_pkey" PRIMARY KEY ("trace_id", "trace_ref")
+);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1939,6 +1939,9 @@ model LiteLLM_Lens {
   id      String @id
   version Int @default(0)
   data    Json
+  due_at  DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone"))
+
+  @@index([due_at], map: "LiteLLM_Lens_due_at_idx")
 }
 
 model LiteLLM_LensRun {
@@ -1974,3 +1977,20 @@ model LiteLLM_LensDataset {
 
   @@id([id, revision])
 }
+
+model LiteLLM_LensSignalConfig {
+  id   String @id
+  data Json
+}
+
+model LiteLLM_LensTraceSignal {
+  trace_id      String
+  trace_ref     String @default("")
+  config_key    String
+  span_count    Int
+  claimed_until DateTime?
+  classified_at DateTime?
+  data          Json
+
+  @@id([trace_id, trace_ref])
+}

litellm-rust/crates/cache/src/semantic.rs

@@ -83,17 +83,16 @@ impl Embedder for PreparedEmbedding {
     }
 }
 
-/// `get_str_from_messages`: every message's text content followed by its search results.
+/// `get_semantic_cache_prompt_from_messages`: every message's text content, including the text of
+/// Messages API `tool_result` blocks, followed by its search results.
 pub fn str_from_messages(messages: &[Value]) -> String {
     let mut text = String::new();
     for message in messages.iter().filter_map(Value::as_object) {
         match message.get("content") {
             Some(Value::String(content)) => text.push_str(content),
-            Some(Value::Array(parts)) => {
-                for part in parts {
-                    if let Some(part_text) = part.get("text").and_then(Value::as_str) {
-                        text.push_str(part_text);
-                    }
+            Some(Value::Array(blocks)) => {
+                for block in blocks {
+                    push_block_text(&mut text, block);
                 }
             }
             _ => {}
@@ -103,6 +102,28 @@ pub fn str_from_messages(messages: &[Value]) -> String {
     text
 }
 
+fn push_block_text(text: &mut String, block: &Value) {
+    if block.get("type").and_then(Value::as_str) != Some("tool_result") {
+        push_text_field(text, block);
+        return;
+    }
+    match block.get("content") {
+        Some(Value::String(result)) => text.push_str(result),
+        Some(Value::Array(blocks)) => {
+            for inner in blocks {
+                push_text_field(text, inner);
+            }
+        }
+        _ => {}
+    }
+}
+
+fn push_text_field(text: &mut String, block: &Value) {
+    if let Some(block_text) = block.get("text").and_then(Value::as_str) {
+        text.push_str(block_text);
+    }
+}
+
 /// The messages prompt Qdrant embeds: `None` when the request carries no messages.
 pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option<String> {
     let messages = context.messages.as_ref()?.as_array()?;
@@ -163,6 +184,10 @@ fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
                 collect_input_text(content, parts);
                 return;
             }
+            if let Some(output) = map.get("output").filter(|output| output.is_array()) {
+                collect_input_text(output, parts);
+                return;
+            }
             for key in ["text", "output", "input_text", "output_text"] {
                 if let Some(Value::String(text)) = map.get(key)
                     && push_trimmed(text, parts)

litellm-rust/crates/cache/tests/semantic.rs

@@ -30,6 +30,31 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
     ]}]),
     "What is this?",
 )]
+#[case::tool_result_string(
+    json!([
+        {"role": "user", "content": "list the files"},
+        {"role": "assistant", "content": [
+            {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}},
+        ]},
+        {"role": "user", "content": [
+            {"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"},
+        ]},
+    ]),
+    "list the filescalc.py test_calc.py",
+)]
+#[case::tool_result_blocks(
+    json!([{"role": "user", "content": [
+        {"type": "tool_result", "tool_use_id": "toolu_1", "content": [
+            {"type": "text", "text": "x = 1"},
+            {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}},
+        ]},
+    ]}]),
+    "x = 1",
+)]
+#[case::tool_result_without_content(
+    json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1"}]}]),
+    "",
+)]
 #[case::missing_null_and_empty_content(
     json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]),
     "",
@@ -166,6 +191,15 @@ fn prompt_from_messages_reads_messages_only(
     ])),
     Some("model dump prompt\ndict prompt\ninline prompt"),
 )]
+#[case::function_call_output_blocks(
+    None,
+    Some(json!([
+        {"role": "user", "content": "update the config"},
+        {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\": \"a\"}"},
+        {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "wrote a"}]},
+    ])),
+    Some("update the config\nwrote a"),
+)]
 #[case::object_content(
     None,
     Some(json!({"content": [{"text": "object content prompt"}]})),

litellm-rust/crates/storage-clickhouse/src/read.rs

@@ -61,6 +61,16 @@ pub async fn execute_read(
     connection: &Connection,
     sql: &str,
     parameters: &BTreeMap<String, Parameter>,
+) -> Result<String, Error> {
+    execute_read_with_limits(client, connection, sql, parameters, READ_LIMITS).await
+}
+
+async fn execute_read_with_limits(
+    client: &Client,
+    connection: &Connection,
+    sql: &str,
+    parameters: &BTreeMap<String, Parameter>,
+    limits: ReadLimits,
 ) -> Result<String, Error> {
     if sql.trim().is_empty() {
         return Err(Error::EmptySql);
@@ -89,12 +99,9 @@ pub async fn execute_read(
         .clear()
         .extend_pairs(existing_pairs)
         .append_pair("readonly", "1")
-        .append_pair("max_result_rows", &READ_LIMITS.result_rows.to_string())
+        .append_pair("max_result_rows", &limits.result_rows.to_string())
         .append_pair("result_overflow_mode", "throw")
-        .append_pair(
-            "max_execution_time",
-            &READ_LIMITS.execution_seconds.to_string(),
-        )
+        .append_pair("max_execution_time", &limits.execution_seconds.to_string())
         .append_pair("wait_end_of_query", "1")
         .append_pair("default_format", "JSON");
 
@@ -122,7 +129,7 @@ pub async fn execute_read(
 
     let mut body = Vec::new();
     while let Some(chunk) = response.chunk().await.map_err(|_| Error::Transport)? {
-        if body.len() + chunk.len() > READ_LIMITS.response_bytes {
+        if body.len() + chunk.len() > limits.response_bytes {
             return Err(Error::ResponseTooLarge);
         }
         body.extend_from_slice(&chunk);
@@ -141,6 +148,7 @@ pub trait Query {
     type Params: Serialize;
     type Row: DeserializeOwned;
 
+    const READ_LIMITS: ReadLimits = crate::read::READ_LIMITS;
     const SQL: &'static str;
 }
 
@@ -159,7 +167,14 @@ pub async fn fetch<Q: Query>(
     connection: &Connection,
     params: &Q::Params,
 ) -> Result<Vec<Q::Row>, Error> {
-    let body = execute_read(client, connection, Q::SQL, &parameters(params)?).await?;
+    let body = execute_read_with_limits(
+        client,
+        connection,
+        Q::SQL,
+        &parameters(params)?,
+        Q::READ_LIMITS,
+    )
+    .await?;
     decode_rows::<Q::Row>(&body)
 }
 
@@ -168,7 +183,14 @@ pub async fn fetch_json<Q: Query>(
     connection: &Connection,
     params: &Q::Params,
 ) -> Result<String, Error> {
-    let body = execute_read(client, connection, Q::SQL, &parameters(params)?).await?;
+    let body = execute_read_with_limits(
+        client,
+        connection,
+        Q::SQL,
+        &parameters(params)?,
+        Q::READ_LIMITS,
+    )
+    .await?;
     decode_rows::<Q::Row>(&body)?;
     Ok(body)
 }

litellm-rust/crates/traces-clickhouse/query/lens_content.sql

@@ -17,6 +17,7 @@ SELECT * FROM (
     FROM otel_traces WHERE {source:String}='traces'
       AND ({all_teams:UInt8}=1 OR TeamId={team:String})
       AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY
       AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String})
       AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId > {cursor:String}
     ORDER BY SpanId LIMIT 1 BY SpanId LIMIT 40
@@ -35,5 +36,6 @@ SELECT * FROM (
     FROM spend_logs FINAL WHERE {source:String}='requests'
       AND ({all_teams:UInt8}=1 OR team_id={team:String})
       AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY
       AND request_id={id:String} AND team_id={record_team:String} LIMIT 1
 )

litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql

@@ -2,6 +2,7 @@ SELECT sum(matches) AS count FROM (
     SELECT count() AS matches FROM otel_traces WHERE {source:String}='traces'
       AND ({all_teams:UInt8}=1 OR TeamId={team:String})
       AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY
       AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String})
       AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId={span:String}
       AND position(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage),{quote:String})>0
@@ -9,6 +10,7 @@ SELECT sum(matches) AS count FROM (
     SELECT count() AS matches FROM spend_logs FINAL WHERE {source:String}='requests'
       AND ({all_teams:UInt8}=1 OR team_id={team:String})
       AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY
       AND request_id={id:String} AND team_id={record_team:String} AND request_id={span:String}
       AND position(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str),{quote:String})>0
 )

litellm-rust/crates/traces-clickhouse/query/lens_sample.sql

@@ -18,10 +18,14 @@ SELECT *, selection_key FROM (
     WHERE {source:String} IN ('traces','both')
       AND ({all_teams:UInt8}=1 OR TeamId={team:String})
       AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      -- The 7 day slack covers spans that started before the window and late ingestion
+      AND Timestamp >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY
       AND (TeamId,ApiKeyHash,TraceId) IN (
           SELECT TeamId,ApiKeyHash,TraceId FROM otel_traces
           WHERE ({all_teams:UInt8}=1 OR TeamId={team:String})
             AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+            AND Timestamp >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY
+            AND Timestamp < fromUnixTimestamp64Milli(toInt64({end:UInt64}))
             AND if(EngineReceivedMs>0,toInt64(EngineReceivedMs),
                 toUnixTimestamp64Milli(Timestamp)+toInt64(intDiv(Duration,1000000))) >= {start:UInt64}
       )
@@ -42,6 +46,8 @@ SELECT *, selection_key FROM (
     WHERE {source:String} IN ('requests','both')
       AND ({all_teams:UInt8}=1 OR team_id={team:String})
       AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND spend_logs.start_time >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY
+      AND spend_logs.start_time < fromUnixTimestamp64Milli(toInt64({end:UInt64}))
       AND if(EngineReceivedMs>0,toInt64(EngineReceivedMs),toUnixTimestamp64Milli(end_time)) >= {start:UInt64}
       AND EngineReceivedMs < {end:UInt64}
       AND toUnixTimestamp64Milli(end_time) < {end:UInt64}
@@ -55,6 +61,8 @@ SELECT *, selection_key FROM (
           SELECT TeamId,ApiKeyHash,LiteLLMRequestId FROM otel_traces
           WHERE ({all_teams:UInt8}=1 OR TeamId={team:String})
             AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) AND LiteLLMRequestId!=''
+            AND Timestamp >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY
+            AND Timestamp < fromUnixTimestamp64Milli(toInt64({end:UInt64}))
       ))
 )
 WHERE ({selected_team:String}='' OR team_id={selected_team:String})

litellm-rust/crates/traces-clickhouse/src/query/lens.rs

@@ -1,4 +1,10 @@
-use litellm_storage_clickhouse::Query;
+use litellm_storage_clickhouse::{Query, ReadLimits};
+
+const SAMPLE_READ_LIMITS: ReadLimits = ReadLimits {
+    result_rows: 10_000,
+    response_bytes: 16 * 1024 * 1024,
+    ..litellm_storage_clickhouse::READ_LIMITS
+};
 
 pub const LENS_QUERIES: [litellm_traces::ReadQuery; 5] = [
     litellm_traces::ReadQuery::Availability,
@@ -188,6 +194,7 @@ impl Query for LensSample {
     type Params = LensSampleParams;
     type Row = LensSampleRow;
 
+    const READ_LIMITS: ReadLimits = SAMPLE_READ_LIMITS;
     const SQL: &'static str = include_str!("../../query/lens_sample.sql");
 }
 
@@ -202,6 +209,7 @@ pub struct LensContentParams {
     pub source: ContentSource,
     pub id: String,
     pub record_team: String,
+    pub start_time: String,
     pub trace_ref: String,
     pub cursor: String,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -245,6 +253,7 @@ pub struct LensEvidenceParams {
     pub source: ContentSource,
     pub id: String,
     pub record_team: String,
+    pub start_time: String,
     pub trace_ref: String,
     pub span: String,
     pub quote: String,

litellm-rust/crates/traces-clickhouse/src/query/number.rs

@@ -99,7 +99,7 @@ mod tests {
     fn content_rejects_unsupported_sources(#[case] source: &str, #[case] valid: bool) {
         let parameters = serde_json::json!({
             "all_teams": 0, "team": "team", "key_hash": "", "source": source, "id": "id",
-            "record_team": "team", "trace_ref": "", "cursor": "", "offset": 0
+            "record_team": "team", "start_time": "", "trace_ref": "", "cursor": "", "offset": 0
         });
         assert_eq!(
             serde_json::from_value::<crate::query::lens::LensContentParams>(parameters).is_ok(),

litellm-rust/crates/traces-clickhouse/tests/load.rs

@@ -0,0 +1,227 @@
+use std::collections::BTreeMap;
+
+use litellm_storage_clickhouse::READ_LIMITS;
+use litellm_traces_clickhouse::{Connection, Parameter, ReadQuery, execute_named_read};
+use rstest::rstest;
+use serde_json::Value;
+
+#[path = "queries/support.rs"]
+#[expect(
+    dead_code,
+    reason = "load tests share the query fixture but do not read through QueryReaders"
+)]
+mod fixtures;
+mod support;
+
+use fixtures::{DATABASE, SeededDatabase, migrated_database};
+use support::TestResult;
+
+const SPANS_PER_DAY: u64 = 2_000;
+
+async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestResult {
+    let count = SPANS_PER_DAY * days;
+    let first_row = SPANS_PER_DAY * first_day;
+    let query = format!(
+        "INSERT INTO {DATABASE}.otel_traces \
+         (Timestamp, TraceId, SpanId, ParentSpanId, SpanName, ServiceName, ObservationType, TeamId, ApiKeyHash, Duration, SpanAttributes) \
+         SELECT now64(9) - toIntervalHour(intDiv(number, {SPANS_PER_DAY}) * 24 + 12 + {first_day} * 24), \
+         if({first_day} = 0, concat('load-', toString(number + {first_row})), 'load-0'), \
+         concat('span-', toString(number + {first_row})), \
+         '', 'span', 'service', 'agent', 'load-team', '', 0, \
+         if({first_day}=0 AND number < {SPANS_PER_DAY}, map('payload', repeat('x', 3000)), map()) \
+         FROM numbers({count})"
+    );
+    fixture
+        .database
+        .client
+        .post(&fixture.database.url)
+        .body(query)
+        .send()
+        .await?
+        .error_for_status()?;
+    Ok(())
+}
+
+async fn trace_start_time(fixture: &SeededDatabase) -> TestResult<String> {
+    let query = format!(
+        "SELECT toString(Timestamp, 'UTC') AS start_time FROM {DATABASE}.otel_traces \
+         WHERE TraceId = 'load-0' LIMIT 1 FORMAT JSON"
+    );
+    let response = fixture
+        .database
+        .client
+        .post(&fixture.database.url)
+        .body(query)
+        .send()
+        .await?
+        .error_for_status()?
+        .text()
+        .await?;
+    let result: Value = serde_json::from_str(&response)?;
+    result["data"][0]["start_time"]
+        .as_str()
+        .map(str::to_owned)
+        .ok_or_else(|| "trace start time missing".into())
+}
+
+fn content_parameters(start_time: &str) -> BTreeMap<String, Parameter> {
+    BTreeMap::from([
+        ("source".into(), Parameter::Text("traces".into())),
+        ("all_teams".into(), Parameter::Integer(0)),
+        ("team".into(), Parameter::Text("load-team".into())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        ("id".into(), Parameter::Text("load-0".into())),
+        ("record_team".into(), Parameter::Text("load-team".into())),
+        ("start_time".into(), Parameter::Text(start_time.into())),
+        ("trace_ref".into(), Parameter::Text(String::new())),
+        ("cursor".into(), Parameter::Text(String::new())),
+        ("offset".into(), Parameter::Integer(1)),
+    ])
+}
+
+async fn content(fixture: &SeededDatabase, start_time: &str, query_id: &str) -> TestResult {
+    let connection = Connection::configured(
+        &format!("{}?query_id={query_id}", fixture.database.url),
+        DATABASE,
+        "default",
+        "",
+    )?;
+    let response = execute_named_read(
+        &fixture.database.client,
+        &connection,
+        ReadQuery::Content,
+        &content_parameters(start_time),
+    )
+    .await?;
+    let result: Value = serde_json::from_str(&response)?;
+    assert!(!result["data"].as_array().ok_or("content rows")?.is_empty());
+    Ok(())
+}
+
+fn sample_parameters(start: u64, end: u64) -> BTreeMap<String, Parameter> {
+    BTreeMap::from([
+        ("source".into(), Parameter::Text("traces".into())),
+        ("all_teams".into(), Parameter::Integer(0)),
+        ("team".into(), Parameter::Text("load-team".into())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        ("start".into(), Parameter::Unsigned(start)),
+        ("end".into(), Parameter::Unsigned(end)),
+        ("agent_name".into(), Parameter::Text(String::new())),
+        ("service".into(), Parameter::Text(String::new())),
+        ("filter_keys".into(), Parameter::Strings(Vec::new())),
+        ("filter_values".into(), Parameter::Strings(Vec::new())),
+        ("selected_team".into(), Parameter::Text(String::new())),
+        ("execution_ids".into(), Parameter::Strings(Vec::new())),
+        ("sample_cap".into(), Parameter::Unsigned(0)),
+        ("sample_percent".into(), Parameter::Integer(100)),
+        ("preview".into(), Parameter::Integer(0)),
+        ("after".into(), Parameter::Text(String::new())),
+        ("limit".into(), Parameter::Unsigned(10_000)),
+        ("offset".into(), Parameter::Unsigned(0)),
+    ])
+}
+
+async fn sample(
+    fixture: &SeededDatabase,
+    start: u64,
+    end: u64,
+    query_id: &str,
+) -> TestResult<(usize, usize)> {
+    let mut url = Connection::configured(&fixture.database.url, DATABASE, "default", "")?
+        .url()
+        .clone();
+    url.query_pairs_mut().append_pair("query_id", query_id);
+    let connection = Connection::parse(url.as_str())?;
+    let response = execute_named_read(
+        &fixture.database.client,
+        &connection,
+        ReadQuery::Sample,
+        &sample_parameters(start, end),
+    )
+    .await?;
+    let result: Value = serde_json::from_str(&response)?;
+    Ok((
+        result["data"].as_array().ok_or("sample rows")?.len(),
+        response.len(),
+    ))
+}
+
+async fn query_read_rows(fixture: &SeededDatabase, query_id: &str) -> TestResult<u64> {
+    fixture
+        .database
+        .client
+        .post(&fixture.database.url)
+        .body("SYSTEM FLUSH LOGS")
+        .send()
+        .await?
+        .error_for_status()?;
+    let response = fixture
+        .database
+        .client
+        .post(&fixture.database.url)
+        .body(format!(
+            "SELECT read_rows FROM system.query_log WHERE type = 'QueryFinish' \
+             AND query_id = '{query_id}' ORDER BY event_time DESC LIMIT 1 FORMAT JSON"
+        ))
+        .send()
+        .await?
+        .error_for_status()?
+        .text()
+        .await?;
+    let result: Value = serde_json::from_str(&response)?;
+    result["data"][0]["read_rows"]
+        .as_u64()
+        .ok_or_else(|| "query log read_rows missing".into())
+}
+
+#[rstest]
+#[tokio::test]
+async fn lens_sample_reads_scale_with_window_not_retention(
+    #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+) -> TestResult {
+    let fixture = migrated_database?;
+    seed_days(&fixture, 0, 8).await?;
+    let now_ms = time::OffsetDateTime::now_utc().unix_timestamp() as u64 * 1000;
+    let start = now_ms - 86_400_000;
+    let end = now_ms + 60_000;
+    let before_id = format!("lens_sample_before_{}", std::process::id());
+    let (before_rows, response_bytes) = sample(&fixture, start, end, &before_id).await?;
+    assert_eq!(before_rows, SPANS_PER_DAY as usize);
+    assert!(response_bytes > READ_LIMITS.response_bytes);
+    let before = query_read_rows(&fixture, &before_id).await?;
+
+    seed_days(&fixture, 8, 24).await?;
+    let after_id = format!("lens_sample_after_{}", std::process::id());
+    let (after_rows, _) = sample(&fixture, start, end, &after_id).await?;
+    assert_eq!(after_rows, SPANS_PER_DAY as usize);
+    let after = query_read_rows(&fixture, &after_id).await?;
+    assert!(
+        after * 100 <= before * 105,
+        "read_rows grew from {before} to {after}"
+    );
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn lens_content_reads_scale_with_trace_not_retention(
+    #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+) -> TestResult {
+    let fixture = migrated_database?;
+    seed_days(&fixture, 0, 8).await?;
+    let start_time = trace_start_time(&fixture).await?;
+    let before_id = format!("lens_content_before_{}", std::process::id());
+    content(&fixture, &start_time, &before_id).await?;
+    let before = query_read_rows(&fixture, &before_id).await?;
+
+    seed_days(&fixture, 8, 24).await?;
+    let after_id = format!("lens_content_after_{}", std::process::id());
+    content(&fixture, &start_time, &after_id).await?;
+    let after = query_read_rows(&fixture, &after_id).await?;
+    println!("lens_content read_rows: before={before}, after={after}");
+    assert!(
+        after * 100 <= before * 105,
+        "read_rows grew from {before} to {after}"
+    );
+    Ok(())
+}

litellm-rust/crates/traces-clickhouse/tests/migrations.rs

@@ -1150,6 +1150,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate(
         ("source".into(), Parameter::Text("traces".into())),
         ("id".into(), Parameter::Text("shared".into())),
         ("record_team".into(), Parameter::Text("team".into())),
+        ("start_time".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(first_ref.into())),
         ("cursor".into(), Parameter::Text(String::new())),
         ("offset".into(), Parameter::Integer(1)),
@@ -1177,6 +1178,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate(
         ("source".into(), Parameter::Text("traces".into())),
         ("id".into(), Parameter::Text("shared".into())),
         ("record_team".into(), Parameter::Text("team".into())),
+        ("start_time".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(first_ref.into())),
         ("span".into(), Parameter::Text("root".into())),
         ("quote".into(), Parameter::Text(opposite.into())),
@@ -1339,7 +1341,7 @@ async fn lens_selection_pages_without_losing_or_repeating_runs(
 #[case::traces("traces", 9)]
 #[case::requests("requests", 3)]
 #[tokio::test]
-async fn lens_content_keeps_original_span_and_request_timestamps(
+async fn lens_content_keeps_original_timestamps_with_start_time_slack(
     #[future(awt)] database: TestResult<ClickHouseDatabase>,
     #[case] source: &str,
     #[case] precision: usize,
@@ -1379,6 +1381,30 @@ async fn lens_content_keeps_original_span_and_request_timestamps(
     )
     .await?;
     let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let start_time_body = execute_read(
+        &database.client,
+        &connection,
+        "SELECT toString(fromUnixTimestamp64Nano({timestamp:Int64})) AS start_time FORMAT JSON",
+        &BTreeMap::from([(
+            "timestamp".into(),
+            Parameter::Integer(root_start + 86_400_000_000_000),
+        )]),
+    )
+    .await?;
+    let start_time: serde_json::Value = serde_json::from_str(&start_time_body)?;
+    let start_time = start_time["data"][0]["start_time"]
+        .as_str()
+        .ok_or("start time missing")?
+        .to_owned();
+    let parsed_time_body = execute_read(
+        &database.client,
+        &connection,
+        "SELECT toString(parseDateTime64BestEffortOrZero({start_time:String}, 9)) AS start_time FORMAT JSON",
+        &BTreeMap::from([("start_time".into(), Parameter::Text(start_time.clone()))]),
+    )
+    .await?;
+    let parsed_time: serde_json::Value = serde_json::from_str(&parsed_time_body)?;
+    assert_eq!(parsed_time["data"][0]["start_time"], start_time);
     let parameters = BTreeMap::from([
         ("source".into(), Parameter::Text(source.into())),
         ("all_teams".into(), Parameter::Integer(0)),
@@ -1386,6 +1412,7 @@ async fn lens_content_keeps_original_span_and_request_timestamps(
         ("record_team".into(), Parameter::Text("team".into())),
         ("key_hash".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(String::new())),
+        ("start_time".into(), Parameter::Text(start_time)),
         ("id".into(), Parameter::Text("run".into())),
         ("cursor".into(), Parameter::Text(String::new())),
         ("offset".into(), Parameter::Integer(1)),
@@ -1453,6 +1480,7 @@ async fn lens_content_keeps_output_visible_after_long_input(
         ("record_team".into(), Parameter::Text("team".into())),
         ("key_hash".into(), Parameter::Text(String::new())),
         ("trace_ref".into(), Parameter::Text(String::new())),
+        ("start_time".into(), Parameter::Text(String::new())),
         ("id".into(), Parameter::Text("request".into())),
         ("cursor".into(), Parameter::Text(String::new())),
         ("offset".into(), Parameter::Integer(1)),

litellm-rust/crates/traces-clickhouse/tests/queries.rs

@@ -3,7 +3,8 @@ use std::collections::BTreeMap;
 use litellm_storage_clickhouse::fetch;
 use litellm_traces::query::named as contracts;
 use litellm_traces_clickhouse::{
-    QueryScope,
+    Connection, InsertTable, Parameter, QueryScope, ReadQuery, execute_named_read, execute_read,
+    insert_rows,
     query::named::{ListTraces, ListTracesParams, TraceSpans, TraceSpansParams},
     query_help, query_sql,
 };
@@ -18,6 +19,112 @@ mod support;
 use fixtures::{SeededDatabase, insert_export, migrated_database, seeded_database};
 use support::TestResult;
 
+#[rstest]
+#[tokio::test]
+async fn lens_sample_keeps_spans_before_window_start_and_excludes_old_only_traces(
+    #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+) -> TestResult {
+    let fixture = migrated_database?;
+    let start_ms = time::OffsetDateTime::now_utc().unix_timestamp() * 1000 - 86_400_000;
+    let end_ms = start_ms + 86_460_000;
+    let rows = [
+        (
+            "late-root",
+            "trace-with-slack",
+            start_ms - 2 * 86_400_000,
+            "",
+        ),
+        (
+            "in-window",
+            "trace-with-slack",
+            start_ms + 1_000,
+            "late-root",
+        ),
+        ("old-span", "trace-too-old", start_ms - 8 * 86_400_000, ""),
+    ]
+    .into_iter()
+    .map(|(span_id, trace_id, timestamp_ms, parent_span_id)| {
+        BTreeMap::from([
+            (
+                "Timestamp".into(),
+                serde_json::json!(timestamp_ms * 1_000_000),
+            ),
+            ("Duration".into(), serde_json::json!(1_000_000)),
+            ("TraceId".into(), serde_json::json!(trace_id)),
+            ("SpanId".into(), serde_json::json!(span_id)),
+            ("ParentSpanId".into(), serde_json::json!(parent_span_id)),
+            ("SpanName".into(), serde_json::json!(span_id)),
+            ("ObservationType".into(), serde_json::json!("agent")),
+            ("TeamId".into(), serde_json::json!("team-lens")),
+            ("ApiKeyHash".into(), serde_json::json!("")),
+        ])
+    })
+    .collect();
+    let writer = Connection::writer(&fixture.database.url)?;
+    insert_rows(
+        &fixture.database.client,
+        &writer,
+        fixtures::DATABASE,
+        InsertTable::OtelTraces,
+        rows,
+    )
+    .await?;
+    let connection =
+        Connection::configured(&fixture.database.url, fixtures::DATABASE, "default", "")?;
+    let parameters = BTreeMap::from([
+        ("source".into(), Parameter::Text("traces".into())),
+        ("all_teams".into(), Parameter::Integer(0)),
+        ("team".into(), Parameter::Text("team-lens".into())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        ("start".into(), Parameter::Unsigned(start_ms as u64)),
+        ("end".into(), Parameter::Unsigned(end_ms as u64)),
+        ("agent_name".into(), Parameter::Text(String::new())),
+        ("service".into(), Parameter::Text(String::new())),
+        ("filter_keys".into(), Parameter::Strings(Vec::new())),
+        ("filter_values".into(), Parameter::Strings(Vec::new())),
+        ("selected_team".into(), Parameter::Text(String::new())),
+        ("execution_ids".into(), Parameter::Strings(Vec::new())),
+        ("sample_cap".into(), Parameter::Unsigned(0)),
+        ("sample_percent".into(), Parameter::Integer(100)),
+        ("preview".into(), Parameter::Integer(0)),
+        ("after".into(), Parameter::Text(String::new())),
+        ("limit".into(), Parameter::Unsigned(10_000)),
+        ("offset".into(), Parameter::Unsigned(0)),
+    ]);
+    let body = execute_named_read(
+        &fixture.database.client,
+        &connection,
+        ReadQuery::Sample,
+        &parameters,
+    )
+    .await?;
+    let result: serde_json::Value = serde_json::from_str(&body)?;
+    let executions = result["data"].as_array().ok_or("sample rows")?;
+    let trace = executions
+        .iter()
+        .find(|row| row["trace_id"] == "trace-with-slack")
+        .ok_or("sampled trace missing")?;
+    let original_start = execute_read(
+        &fixture.database.client,
+        &connection,
+        "SELECT toString(fromUnixTimestamp64Nano({timestamp:Int64})) AS start_time FORMAT JSON",
+        &BTreeMap::from([(
+            "timestamp".into(),
+            Parameter::Integer((start_ms - 2 * 86_400_000) * 1_000_000),
+        )]),
+    )
+    .await?;
+    let original_start: serde_json::Value = serde_json::from_str(&original_start)?;
+    assert_eq!(trace["span_count"].as_u64(), Some(2));
+    assert_eq!(trace["start_time"], original_start["data"][0]["start_time"]);
+    assert!(
+        !executions
+            .iter()
+            .any(|row| row["trace_id"] == "trace-too-old")
+    );
+    Ok(())
+}
+
 #[derive(Clone, Copy, strum::AsRefStr)]
 #[strum(serialize_all = "snake_case")]
 enum ScopeCase {

litellm/caching/caching.py

@@ -71,6 +71,17 @@ class CacheMode(str, Enum):
 
 
 #### LiteLLM.Completion / Embedding Cache ####
+def _request_message_count(kwargs: Mapping[str, object]) -> int:
+    """Chat and Messages API `messages`, else Responses API `input` items; embedding `input` strings count as none"""
+    messages: Final = kwargs.get("messages")
+    if isinstance(messages, list):
+        return len(messages)
+    input_items: Final = kwargs.get("input")
+    if not isinstance(input_items, list):
+        return 0
+    return sum(1 for item in input_items if isinstance(item, (Mapping, BaseModel)))
+
+
 class Cache:
     def __init__(
         self,
@@ -119,6 +130,7 @@ class Cache:
         semantic_cache_embedding_max_input_tokens: int | None = None,
         semantic_cache_embedding_timeout: float | None = None,
         semantic_cache_scope: str = SemanticCacheScope.KEY.value,
+        max_messages: int | None = 4,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -148,6 +160,7 @@ class Cache:
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
             semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
             semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
+            max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -298,6 +311,7 @@ class Cache:
         self.ttl = ttl
         self.mode: CacheMode = mode or CacheMode.default_on
         self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
+        self.max_messages: int | None = max_messages
 
         if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
             self.ttl = default_in_memory_ttl
@@ -933,7 +947,10 @@ class Cache:
 
         If cache is default_on then this is True
         If cache is default_off then this is only true when user has opted in to use cache
+        Always False once the request carries more than `max_messages` messages
         """
+        if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages:
+            return False
         if self.mode == CacheMode.default_on:
             return True
 

litellm/caching/qdrant_semantic_cache.py

@@ -24,7 +24,7 @@ from litellm.constants import (
 )
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    get_str_from_messages,
+    get_semantic_cache_prompt_from_messages,
 )
 from litellm.types.utils import EmbeddingResponse
 
@@ -286,7 +286,7 @@ class QdrantSemanticCache(BaseCache):
 
         # get the prompt
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # create an embedding for prompt
         embedding_response: Final = cast(
@@ -325,7 +325,7 @@ class QdrantSemanticCache(BaseCache):
 
         # get the messages
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # convert to embedding
         embedding_response: Final = cast(
@@ -400,7 +400,7 @@ class QdrantSemanticCache(BaseCache):
 
         # get the prompt
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
         embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
 
         # get the embedding
@@ -435,7 +435,7 @@ class QdrantSemanticCache(BaseCache):
 
         # get the messages
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
 

litellm/caching/redis_semantic_cache.py

@@ -21,7 +21,7 @@ from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    get_str_from_messages,
+    get_semantic_cache_prompt_from_messages,
 )
 from litellm.types.utils import EmbeddingResponse
 
@@ -263,7 +263,7 @@ class RedisSemanticCache(BaseCache):
         """
         messages: Final = kwargs.get("messages")
         if messages:
-            return get_str_from_messages(messages)
+            return get_semantic_cache_prompt_from_messages(messages)
 
         if "input" not in kwargs:
             return None
@@ -274,7 +274,7 @@ class RedisSemanticCache(BaseCache):
         return prompt or None
 
     @classmethod
-    def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None:
+    def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None:  # noqa: C901  # one branch per Responses input shape
         value = cls._coerce_response_input_value(value)
         if value is None:
             return
@@ -296,6 +296,11 @@ class RedisSemanticCache(BaseCache):
                 cls._collect_responses_input_text(content, prompt_parts)
                 return
 
+            output = value.get("output")
+            if isinstance(output, list):
+                cls._collect_responses_input_text(output, prompt_parts)
+                return
+
             for text_key in ("text", "output", "input_text", "output_text"):
                 text_value = value.get(text_key)
                 if isinstance(text_value, str):

litellm/integrations/anthropic_cache_control_hook.py

@@ -1029,7 +1029,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
         if openai_dialect and breakpoints_added > 0:
-            kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
+            kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="implicit"))
         if remaining:
             kwargs["cache_control_injection_points"] = remaining
         return messages, system

litellm/integrations/custom_guardrail.py

@@ -24,6 +24,7 @@ from litellm.types.guardrails import (
     DynamicGuardrailParams,
     GuardrailEventHooks,
     LitellmParams,
+    LoggingOnlyScope,
     Mode,
 )
 from litellm.types.llms.openai import AllMessageValues
@@ -180,6 +181,7 @@ class CustomGuardrail(CustomLogger):
     use_native_lifecycle_hooks: ClassVar[bool] = False
 
     records_own_guardrail_information: ClassVar[bool] = False
+    logging_only_scope: LoggingOnlyScope | None
 
     timeout: float | httpx.Timeout | None = None
 
@@ -256,6 +258,7 @@ class CustomGuardrail(CustomLogger):
         self.run_in_parallel: bool = run_in_parallel
         self.scan_raw_request: bool = scan_raw_request
         self.only_scan_new_messages: bool = only_scan_new_messages
+        self.logging_only_scope = None
         if timeout is not None:
             self.timeout = timeout
 
@@ -817,6 +820,13 @@ class CustomGuardrail(CustomLogger):
     def uses_apply_guardrail_interface(self) -> bool:
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
+    @classmethod
+    def supports_logging_only_scope(cls) -> bool:
+        return (
+            cls.apply_guardrail is not CustomGuardrail.apply_guardrail
+            and cls.async_logging_hook is CustomGuardrail.async_logging_hook
+        )
+
     def _deployment_hook_target(self) -> "CustomLogger":
         if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
@@ -948,7 +958,7 @@ class CustomGuardrail(CustomLogger):
         result: object,
         call_type: str,
     ) -> tuple[dict, object]:
-        """logging_only: run apply_guardrail on copies of the logged request/response and record the verdict."""
+        """logging_only: scan copies of the logged request and/or response according to logging_only_scope."""
         from litellm.llms import get_guardrail_translation_mapping
 
         if not self.uses_apply_guardrail_interface():
@@ -995,6 +1005,28 @@ class CustomGuardrail(CustomLogger):
             "standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]},
         }, result
 
+    def _copy_scratch_request_fields(
+        self,
+        kwargs: Mapping[str, object],
+    ) -> tuple[object, object] | None:
+        optional_params: Final = kwargs.get("optional_params")
+        try:
+            return (
+                copy.deepcopy(kwargs.get("messages") or kwargs.get("input")),
+                copy.deepcopy(optional_params.get("tools") if isinstance(optional_params, Mapping) else None),
+            )
+        except Exception as e:
+            if self.logging_only_scope == "output":
+                return None
+            if self.logging_only_scope == "both":
+                verbose_logger.warning(
+                    "Guardrail %s: logging_only request copy failed, skipping request scan: %s",
+                    self.guardrail_name,
+                    e,
+                )
+                return None
+            raise
+
     async def _scan_logged_call(
         self,
         kwargs: dict,
@@ -1003,18 +1035,25 @@ class CustomGuardrail(CustomLogger):
         output_translation: "BaseTranslation",
         scratch_metadata: dict,
     ) -> None:
-        optional_params: Final = kwargs.get("optional_params") or {}
-        scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input"))
+        scratch_fields: Final = self._copy_scratch_request_fields(kwargs)
+        scratch_input, scratch_tools = scratch_fields or (None, None)
         scratch_request: Final = {
             "model": kwargs.get("model"),
             "messages": scratch_input,
             "input": scratch_input,
-            "tools": copy.deepcopy(optional_params.get("tools")),
+            "tools": scratch_tools,
             "litellm_call_id": kwargs.get("litellm_call_id"),
             "metadata": scratch_metadata,
         }
-        await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
-        if response is None:
+        if self.logging_only_scope != "output" and scratch_fields is not None:
+            if self.logging_only_scope == "both":
+                try:
+                    await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
+                except Exception as e:  # noqa: BLE001  # one direction's scan failure must not drop the other direction's verdict
+                    verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e)
+            else:
+                await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
+        if response is None or self.logging_only_scope == "input":
             return
         await output_translation.process_output_response(
             response=copy.deepcopy(response), guardrail_to_apply=self, request_data=scratch_request

litellm/integrations/custom_logger.py

@@ -909,7 +909,8 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         """
         import litellm
         from litellm import Choices, Message, ModelResponse
-        from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit
+        from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
+        from litellm.litellm_core_utils.redact_messages import redacted_litellm_params
 
         turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
         excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
@@ -918,9 +919,15 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         if turn_off_message_logging is False and not excluded_fields:
             return model_call_details
 
+        params: Final = model_call_details.get("litellm_params")
+        redacted_params: Final = (
+            MappingProxyType({"litellm_params": redacted_litellm_params(params)})
+            if turn_off_message_logging and isinstance(params, Mapping)
+            else EMPTY_MAPPING
+        )
         standard_logging_object: Final = model_call_details.get("standard_logging_object")
         if standard_logging_object is None:
-            return model_call_details.copy()
+            return {**model_call_details, **redacted_params}
 
         # Make a copy of just the standard_logging_object to avoid modifying the original
         standard_logging_object_copy: Final = {
@@ -960,13 +967,6 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
                     model_response_dict: Final = model_response.model_dump()
                     standard_logging_object_copy["response"] = model_response_dict
 
-        params: Final = model_call_details.get("litellm_params")
-        request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None
-        redacted_params: Final = (
-            MappingProxyType({"litellm_params": {**params, "proxy_server_request": without_classifier_audit(request)}})
-            if turn_off_message_logging and isinstance(params, dict) and isinstance(request, dict)
-            else EMPTY_MAPPING
-        )
         return {
             **model_call_details,
             **redacted_params,

litellm/integrations/otel/model/spans.py

@@ -353,6 +353,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_LensRun",
         "LiteLLM_LensReview",
         "LiteLLM_LensWorker",
+        "LiteLLM_LensSignalConfig",
+        "LiteLLM_LensTraceSignal",
     )
 )
 PRISMA_RELATIONS: Final[frozenset[str]] = _PRISMA_MODELS | _PRISMA_VIEWS

litellm/litellm_core_utils/logging_utils.py

@@ -43,7 +43,7 @@ Helper utils used for logging callbacks
 
 # Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
 # Captures: group(1)=mime_type, group(2)=base64_payload
-_DATA_URI_RE: Final = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)")
+_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
 
 # Maximum nesting depth for _truncate_base64_in_value to guard against
 # pathological payloads. OpenAI message format is typically 3-4 levels deep.

litellm/litellm_core_utils/logging_worker.py

@@ -23,6 +23,19 @@ from litellm.constants import (
     MAX_TIME_TO_CLEAR_QUEUE,
 )
 
+_CALLBACK_DEADLINE: Final[contextvars.ContextVar[float | None]] = contextvars.ContextVar(
+    "logging_callback_deadline", default=None
+)
+
+
+def optional_callback_budget(maximum: float, *, fraction: float = 0.25) -> float:
+    deadline: Final = _CALLBACK_DEADLINE.get()
+    return (
+        maximum
+        if deadline is None
+        else max(0.0, min(maximum, (deadline - asyncio.get_running_loop().time()) * fraction))
+    )
+
 
 def _coroutine_name(coroutine: Coroutine) -> str:
     return getattr(coroutine, "__qualname__", None) or getattr(coroutine, "__name__", None) or type(coroutine).__name__
@@ -100,12 +113,20 @@ class LoggingWorker:
         return len(revived)
 
     def _run_coroutine_silently(self, loop: asyncio.AbstractEventLoop, coroutine: Coroutine) -> bool:
+        token: Final = _CALLBACK_DEADLINE.set(loop.time() + self.timeout)
         try:
             loop.run_until_complete(asyncio.wait_for(coroutine, timeout=self.timeout))
         except (Exception, asyncio.CancelledError):  # noqa: BLE001  # atexit flush must never break the user's program
             return False
+        finally:
+            _CALLBACK_DEADLINE.reset(token)
         return True
 
+    def _create_callback_task(self, task: LoggingTask) -> asyncio.Task[object]:
+        context: Final = task["context"].copy()
+        context.run(_CALLBACK_DEADLINE.set, asyncio.get_running_loop().time() + self.timeout)
+        return context.run(asyncio.create_task, task["coroutine"])
+
     @staticmethod
     def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
         """Pop every task still queued, without awaiting them, so they can be moved to another queue."""
@@ -172,7 +193,7 @@ class LoggingWorker:
         try:
             if self._queue is not None:
                 # Run the coroutine in its original context
-                callback_task: Final = task["context"].run(asyncio.create_task, task["coroutine"])
+                callback_task: Final = self._create_callback_task(task)
                 try:
                     await asyncio.wait_for(callback_task, timeout=self.timeout)
                 except asyncio.TimeoutError as e:
@@ -424,7 +445,7 @@ class LoggingWorker:
 
         try:
             await asyncio.wait_for(
-                task["context"].run(asyncio.create_task, task["coroutine"]),
+                self._create_callback_task(task),
                 timeout=self.timeout,
             )
         except Exception:
@@ -517,7 +538,7 @@ class LoggingWorker:
                 # Await the coroutine to properly execute and avoid "never awaited" warnings
                 try:
                     await asyncio.wait_for(
-                        task["context"].run(asyncio.create_task, task["coroutine"]),
+                        self._create_callback_task(task),
                         timeout=self.timeout,
                     )
                 except Exception:

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -192,6 +192,33 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str:
     return text
 
 
+def get_semantic_cache_prompt_from_messages(messages: Sequence[Mapping[str, object]]) -> str:
+    """
+    The text a semantic cache embeds for a request: `get_str_from_messages` plus the text inside
+    Messages API `tool_result` blocks, so a tool turn does not embed identically to the turn before it
+    """
+    return "".join(
+        _semantic_cache_content_text(message.get("content"))
+        + extract_search_results_text(message.get("search_results"))
+        for message in messages
+    )
+
+
+def _semantic_cache_content_text(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return ""
+    return "".join(_semantic_cache_block_text(block) for block in content if isinstance(block, Mapping))
+
+
+def _semantic_cache_block_text(block: Mapping[str, object]) -> str:
+    if block.get("type") == "tool_result":
+        return _semantic_cache_content_text(block.get("content"))
+    text: Final = block.get("text")
+    return text if isinstance(text, str) else ""
+
+
 def is_non_content_values_set(message: AllMessageValues) -> bool:
     ignore_keys: Final = ["content", "role", "name"]
     return any(message.get(key, None) is not None for key in message if key not in ignore_keys)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -5188,7 +5188,7 @@ def function_call_prompt(
     messages: list[dict[str, object]],
     functions: list[object],
 ) -> list[dict[str, object]]:
-    function_prompt = """Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}} The following functions are available to you:"""
+    function_prompt = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
     for function in functions:
         function_prompt += f"""\n{function}\n"""
 

litellm/litellm_core_utils/realtime_streaming.py

@@ -896,8 +896,8 @@ class RealTimeStreaming:
                 # clientContent / cancel messages are sent.
                 if pre_block_backend_message is not None:
                     await self._send_to_backend(pre_block_backend_message)
-                # Cancel any in-progress LLM response (e.g. VAD auto-response).
-                await self._send_to_backend(json.dumps({"type": "response.cancel"}))
+                if not self._is_transcription_session:
+                    await self._send_to_backend(json.dumps({"type": "response.cancel"}))
                 # Send the policy violation hint (shows as small gray status text in UI).
                 await self.websocket.send_text(
                     json.dumps(
@@ -911,25 +911,26 @@ class RealTimeStreaming:
                         }
                     )
                 )
-                # Ask the LLM to voice the exact guardrail message so the
-                # user hears it as audio in voice sessions (not just text).
-                guardrail_prompt = (
-                    f"Say exactly the following message to the user, word for word, "
-                    f"do not add anything else: {error_msg}"
-                )
-                await self._send_to_backend(
-                    json.dumps(
-                        {
-                            "type": "conversation.item.create",
-                            "item": {
-                                "type": "message",
-                                "role": "user",
-                                "content": [{"type": "input_text", "text": guardrail_prompt}],
-                            },
-                        }
+                if not self._is_transcription_session:
+                    # Ask the LLM to voice the exact guardrail message so the
+                    # user hears it as audio in voice sessions (not just text).
+                    guardrail_prompt = (
+                        f"Say exactly the following message to the user, word for word, "
+                        f"do not add anything else: {error_msg}"
                     )
-                )
-                await self._send_to_backend(json.dumps({"type": "response.create"}))
+                    await self._send_to_backend(
+                        json.dumps(
+                            {
+                                "type": "conversation.item.create",
+                                "item": {
+                                    "type": "message",
+                                    "role": "user",
+                                    "content": [{"type": "input_text", "text": guardrail_prompt}],
+                                },
+                            }
+                        )
+                    )
+                    await self._send_to_backend(json.dumps({"type": "response.create"}))
 
                 self._violation_count += 1
                 end_session_after: int | None = getattr(callback, "end_session_after_n_fails", None)
@@ -1070,18 +1071,14 @@ class RealTimeStreaming:
             self.store_message(event_obj)
             await self.websocket.send_text(self._event_to_client_json(event_obj))
 
-            # Transcription-only sessions (e.g. gpt-realtime-whisper) have no
-            # assistant turn: capture audio-duration usage for cost and never
-            # trigger response.create.
             if self._is_transcription_session:
                 self._capture_transcription_usage(event_obj)
-                return True
 
             blocked: Final = await self.run_realtime_guardrails(
                 transcript,
                 item_id=event_obj.get("item_id"),
             )
-            if not blocked:
+            if not blocked and not self._is_transcription_session:
                 await self._send_to_backend(json.dumps({"type": "response.create"}))
             return True
         return False

litellm/litellm_core_utils/redact_messages.py

@@ -11,6 +11,7 @@ import asyncio
 import copy
 import inspect
 from collections.abc import Mapping
+from dataclasses import replace
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
@@ -26,6 +27,7 @@ from litellm.llms.vertex_ai.common_utils import (
     redact_vertex_ai_metadata_from_logged_object,
 )
 from litellm.secret_managers.main import str_to_bool
+from litellm.types.router import BaselineRouteStamp
 from litellm.types.utils import StandardCallbackDynamicParams
 
 if TYPE_CHECKING:
@@ -252,6 +254,26 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
             _redact_choice_content(choice)
 
 
+def _redacted_baseline_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]:
+    route: Final = metadata.get("_autorouter_baseline_route")
+    if not isinstance(route, BaselineRouteStamp):
+        return metadata
+    return {**metadata, "_autorouter_baseline_route": replace(route, request_parameters=None)}
+
+
+def redacted_litellm_params(params: Mapping[str, object]) -> dict[str, object]:
+    request: Final = params.get("proxy_server_request")
+    return {
+        **params,
+        **{
+            key: _redacted_baseline_metadata(value)
+            for key, value in params.items()
+            if key in ("metadata", "litellm_metadata") and isinstance(value, Mapping)
+        },
+        **({"proxy_server_request": without_classifier_audit(request)} if isinstance(request, Mapping) else {}),
+    }
+
+
 def perform_redaction(model_call_details: dict, result, redact_streaming_responses: bool = True):
     """
     Performs the actual redaction on the logging object and result.
@@ -262,9 +284,8 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
     """
     # Redact model_call_details
     params: Final = model_call_details.get("litellm_params")
-    request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None
-    if isinstance(params, dict) and isinstance(request, Mapping):
-        model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)}
+    if isinstance(params, Mapping):
+        model_call_details["litellm_params"] = redacted_litellm_params(params)
     model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""

litellm/llms/anthropic/pass_through/messages/handler.py

@@ -15,9 +15,6 @@ import litellm
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.common_utils import (
-    flatten_unencrypted_web_search_results_in_anthropic_messages,
-    sanitize_tool_use_ids_in_anthropic_messages,
-    strip_empty_content_blocks_from_anthropic_messages,
     strip_provider_specific_fields_from_anthropic_messages,
 )
 from litellm.llms.base_llm.anthropic_messages.transformation import (
@@ -36,9 +33,8 @@ from litellm.utils import ProviderConfigManager, client
 
 from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
 from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
-from ..utils import is_reasoning_auto_summary_enabled
 from .interceptors import get_messages_interceptors
-from .utils import AnthropicMessagesRequestUtils, mock_response
+from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages
 
 __all__ = ("anthropic_messages", "anthropic_messages_handler")
 
@@ -251,28 +247,7 @@ async def anthropic_messages(
 
     Runs the empty-content-block sanitizer before any backend dispatch.
     """
-    # Anthropic's API rejects requests containing empty / whitespace-only
-    # text content blocks ("messages: text content blocks must be
-    # non-empty") and empty thinking blocks ("each thinking block must
-    # contain thinking").  Multi-turn tool-use clients (e.g. Claude Code)
-    # routinely loop assistant responses that contain such blocks — an empty
-    # text block alongside tool_use, or an empty thinking block from a turn
-    # a non-Anthropic reasoning model served through the bridge — back as
-    # conversation history, which then causes the next /v1/messages call to
-    # 400.  /v1/chat/completions already handles this in
-    # anthropic_messages_pt; sanitize the native Anthropic Messages path
-    # here for the same guarantee.  See #22930.
-    messages = strip_empty_content_blocks_from_anthropic_messages(messages)
-    # Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry
-    # ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
-    messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
-    messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
-
-    from litellm.integrations.anthropic_cache_control_hook import (
-        AnthropicCacheControlHook,
-    )
-
-    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
+    messages, system = prepare_native_messages(
         messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
     )
 
@@ -454,23 +429,15 @@ def anthropic_messages_handler(
     """
     from litellm.types.utils import LlmProviders
 
-    # Sanitize empty text blocks so the sync entry point
-    # (litellm.messages.create -> anthropic_messages_handler) gets the same
-    # protection as the async wrapper. The async wrapper already sanitized and
-    # does not reassign messages before dispatch, so it sets
-    # ``_litellm_messages_presanitized`` to skip this redundant second
-    # full-messages scan. Pop it so it never leaks into provider params.
-    if not kwargs.pop("_litellm_messages_presanitized", False):
-        messages = strip_empty_content_blocks_from_anthropic_messages(messages)
-        messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
-        messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
-
-    from litellm.integrations.anthropic_cache_control_hook import (
-        AnthropicCacheControlHook,
-    )
-
-    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
-        messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
+    messages, system = prepare_native_messages(
+        messages,
+        system,
+        kwargs,
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        tools=tools,
+        api_base=api_base,
+        presanitized=bool(kwargs.pop("_litellm_messages_presanitized", False)),
     )
 
     metadata = validate_anthropic_api_metadata(metadata)
@@ -645,14 +612,6 @@ def anthropic_messages_handler(
             custom_llm_provider=custom_llm_provider,
         )
     )
-    if is_reasoning_auto_summary_enabled():
-        thinking_param: Final = anthropic_messages_optional_request_params.get("thinking")
-        if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled":
-            anthropic_messages_optional_request_params["thinking"] = {
-                **thinking_param,
-                "display": "summarized",
-            }
-
     resolved_api_base: Final = (
         dynamic_api_base
         if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base()

litellm/llms/anthropic/pass_through/messages/utils.py

@@ -2,6 +2,15 @@ from collections.abc import Iterable, Mapping, Sequence
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints
 
+from pydantic import JsonValue
+
+from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+from litellm.llms.anthropic.common_utils import (
+    flatten_unencrypted_web_search_results_in_anthropic_messages,
+    sanitize_tool_use_ids_in_anthropic_messages,
+    strip_empty_content_blocks_from_anthropic_messages,
+)
+from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled
 from litellm.types.llms.anthropic import (
     AnthropicMessagesRequestOptionalParams,
     AnthropicStopDetails,
@@ -119,8 +128,40 @@ def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMe
     return ChatCompletionSystemMessage(role="system", content=system)
 
 
+def prepare_native_messages(
+    messages: list[dict[str, JsonValue]],
+    system: str | list[dict[str, JsonValue]] | None,
+    kwargs: dict[str, object],
+    *,
+    model: str,
+    custom_llm_provider: str | None = None,
+    tools: list[dict[str, JsonValue]] | None = None,
+    api_base: str | None = None,
+    presanitized: bool = False,
+) -> tuple[list[dict[str, JsonValue]], str | list[dict[str, JsonValue]] | None]:
+    normalized: Final = (
+        messages
+        if presanitized
+        else flatten_unencrypted_web_search_results_in_anthropic_messages(
+            sanitize_tool_use_ids_in_anthropic_messages(strip_empty_content_blocks_from_anthropic_messages(messages))
+        )
+    )
+    return cast(  # cast-ok: legacy normalizers and injection preserve the JSON message and system shapes
+        tuple[list[dict[str, JsonValue]], str | list[dict[str, JsonValue]] | None],
+        AnthropicCacheControlHook.maybe_inject_cache_control(
+            normalized,
+            system,
+            kwargs,
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            tools=tools,
+            api_base=api_base,
+        ),
+    )
+
+
 @lru_cache(maxsize=1)
-def _anthropic_messages_optional_param_keys() -> frozenset[str]:
+def anthropic_messages_optional_param_keys() -> frozenset[str]:
     """
     Valid AnthropicMessagesRequestOptionalParams keys.
 
@@ -152,7 +193,7 @@ class AnthropicMessagesRequestUtils:
         Returns:
             AnthropicMessagesRequestOptionalParams instance with only the valid parameters
         """
-        valid_keys: Final = _anthropic_messages_optional_param_keys()
+        valid_keys: Final = anthropic_messages_optional_param_keys()
         filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None}
         if model is not None:
             from litellm.llms.anthropic.chat.transformation import AnthropicConfig
@@ -174,6 +215,13 @@ class AnthropicMessagesRequestUtils:
                         drop_params=drop_params,
                         output_key=param,
                     )
+        if is_reasoning_auto_summary_enabled():
+            thinking_param: Final = filtered_params.get("thinking")
+            if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled":
+                return cast(
+                    AnthropicMessagesRequestOptionalParams,
+                    {**filtered_params, "thinking": {**thinking_param, "display": "summarized"}},
+                )
         return cast(AnthropicMessagesRequestOptionalParams, filtered_params)
 
 

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -5,6 +5,7 @@ import hashlib
 import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
+from functools import reduce
 from itertools import accumulate, groupby
 from types import MappingProxyType
 from typing import Annotated, Final, Literal, Protocol, TypeAlias
@@ -13,20 +14,32 @@ import httpx
 from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
 
 import litellm
-from litellm.llms.anthropic.common_utils import AnthropicModelInfo, is_anthropic_oauth_key
+from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
+from litellm.llms.anthropic.common_utils import (
+    AnthropicModelInfo,
+    is_anthropic_oauth_key,
+    strip_provider_specific_fields_from_anthropic_messages,
+)
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
 from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES
 from litellm.llms.anthropic.pass_through.messages.transformation import (
     DEFAULT_ANTHROPIC_API_VERSION,
     AnthropicMessagesConfig,
 )
+from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils, prepare_native_messages
+from litellm.router_utils.baseline_request import (
+    BASELINE_PARAMETERS,
+    capture_baseline_parameters,
+)
 from litellm.types.llms.base import LiteLLMBaseModel
-from litellm.types.router import LiteLLM_Params
+from litellm.types.router import GenericLiteLLMParams, LiteLLM_Params
 from litellm.types.utils import ModelResponse
 from litellm.utils import supports_thinking_cache_preservation
 
 _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
 _HEADERS: Final = TypeAdapter(dict[str, str])
+_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]])
+_SYSTEM: Final = TypeAdapter(str | list[dict[str, JsonValue]] | None)
 _counter: Final = AnthropicCountTokensHandler()
 
 
@@ -325,7 +338,7 @@ def _entry_fingerprint(fingerprint: str, ttl_seconds: int) -> str:
 
 def parse_cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan | UnsupportedCachePlan:
     try:
-        request: Final = _PlanRequest.model_validate(body)
+        request: Final = _PlanRequest.model_validate(dict(body))
         positions: Final = _positions(body)
     except ValidationError:
         return UnsupportedCachePlan("unsupported_prompt_shape")
@@ -618,13 +631,56 @@ def resolve_baseline_prediction_target(params: LiteLLM_Params) -> NativePredicti
     return _resolve_prediction_target(params, allow_configured_endpoint=True)
 
 
+def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> Mapping[str, JsonValue] | None:
+    parameters: Final = capture_baseline_parameters(request)
+    if parameters is None:
+        return None
+    source: Final = {**parameters, "messages": request.get("messages"), "stream": request.get("stream", False)}
+    try:
+        owned: Final = _JSON_OBJECT.validate_python(source)
+        context: Final = {**{k: v for k, v in request.items() if k not in ("metadata", "litellm_metadata")}, **owned}
+        resolved_model: Final = litellm.get_llm_provider(model=model, custom_llm_provider="anthropic")[0]
+        messages, system = prepare_native_messages(
+            _MESSAGES.validate_python(owned.get("messages")),
+            _SYSTEM.validate_python(owned.get("system")),
+            context,
+            model=resolved_model,
+            custom_llm_provider="anthropic",
+            tools=_MESSAGES.validate_python(owned.get("tools") or []),
+        )
+        options: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
+            {**owned, "system": system},
+            model=resolved_model,
+            custom_llm_provider="anthropic",
+            drop_params=owned.get("drop_params") is True,
+        )
+        filtered: Final = reduce(
+            delete_nested_value,
+            TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
+            dict(options),
+        )
+        body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
+            model=resolved_model,
+            messages=strip_provider_specific_fields_from_anthropic_messages(messages),
+            anthropic_messages_optional_request_params=filtered,
+            litellm_params=GenericLiteLLMParams(),
+            headers={},
+        )
+        return MappingProxyType(_JSON_OBJECT.validate_python(body))
+    except Exception:  # noqa: BLE001  # an unsupported hypothetical request is unavailable, never an inference failure
+        return None
+
+
 def _resolve_prediction_target(
     params: LiteLLM_Params,
     *,
     allow_configured_endpoint: bool,
 ) -> NativePredictionTarget | UnsupportedPredictionTarget:
     configured_options: Final = frozenset(params.model_dump(exclude_defaults=True, exclude_none=True))
-    if configured_options - _DEPLOYMENT_OPTIONS:
+    allowed: Final = (
+        _DEPLOYMENT_OPTIONS | frozenset(BASELINE_PARAMETERS) if allow_configured_endpoint else _DEPLOYMENT_OPTIONS
+    )
+    if configured_options - allowed:
         return UnsupportedPredictionTarget("unsupported_deployment_configuration")
     api_base: Final = AnthropicModelInfo.get_api_base(params.api_base)
     if not allow_configured_endpoint and api_base not in (

litellm/llms/base_llm/chat/transformation.py

@@ -65,6 +65,26 @@ class BaseLLMException(Exception):
         super().__init__(self.message)  # Call the base class constructor with the parameters it needs
 
 
+_NO_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
+
+
+def with_attribution_headers(
+    attribution_headers: Mapping[str, str],
+    headers: dict[str, str] | None,  # mutable-ok: returned as-is when there is nothing to add
+) -> dict[str, str] | None:  # mutable-ok: becomes the request's outbound headers
+    """
+    `headers` plus any attribution header the caller didn't already set (names
+    compared case-insensitively). Builds a new dict; `headers` is never mutated.
+    """
+    if not attribution_headers:
+        return headers
+    caller_names: Final = {name.lower() for name in headers or {}}
+    return {
+        **{name: value for name, value in attribution_headers.items() if name.lower() not in caller_names},
+        **(headers or {}),
+    }
+
+
 class BaseConfig(ABC):
     def __init__(self):
         pass
@@ -89,6 +109,15 @@ class BaseConfig(ABC):
             and not callable(v)  # Filter out any callable objects including mocks
         }
 
+    def get_attribution_headers(self) -> Mapping[str, str]:
+        """
+        Headers that tell the provider a request came through LiteLLM.
+
+        Sent by default on every request; a caller header with the same name
+        (any casing) wins. Override in a provider config to opt in.
+        """
+        return _NO_ATTRIBUTION_HEADERS
+
     def get_json_schema_from_pydantic_object(self, response_format: type[BaseModel] | dict | None) -> dict | None:
         return type_to_response_format_param(response_format=response_format)
 

litellm/llms/novita/chat/transformation.py

@@ -6,11 +6,20 @@ Calls done in OpenAI/openai.py as Novita AI is openai-compatible.
 Docs: https://novita.ai/docs/guides/llm-api
 """
 
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
 from ....types.llms.openai import AllMessageValues
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
+_NOVITA_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"X-Novita-Source": "litellm"})
+
 
 class NovitaConfig(OpenAIGPTConfig):
+    def get_attribution_headers(self) -> Mapping[str, str]:
+        return _NOVITA_ATTRIBUTION_HEADERS
+
     def validate_environment(
         self,
         headers: dict,
@@ -27,5 +36,6 @@ class NovitaConfig(OpenAIGPTConfig):
             )
         headers["Authorization"] = f"Bearer {api_key}"
         headers["Content-Type"] = "application/json"
-        headers["X-Novita-Source"] = "litellm"
+        if not any(name.lower() == "x-novita-source" for name in headers):
+            headers["X-Novita-Source"] = "litellm"
         return headers

litellm/llms/perplexity/chat/transformation.py

@@ -2,6 +2,8 @@
 Translate from OpenAI's `/v1/chat/completions` to Perplexity's `/v1/chat/completions`
 """
 
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
@@ -17,12 +19,17 @@ from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
+_PERPLEXITY_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"X-Pplx-Integration": "litellm"})
+
 
 class PerplexityChatConfig(OpenAIGPTConfig):
     @property
     def custom_llm_provider(self) -> str | None:
         return "perplexity"
 
+    def get_attribution_headers(self) -> Mapping[str, str]:
+        return _PERPLEXITY_ATTRIBUTION_HEADERS
+
     def _get_openai_compatible_provider_info(
         self, api_base: str | None, api_key: str | None
     ) -> tuple[str | None, str | None]:

litellm/main.py

@@ -118,6 +118,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
 from litellm.llms.base_llm.base_model_iterator import (
     convert_model_response_to_streaming,
 )
+from litellm.llms.base_llm.chat.transformation import with_attribution_headers
 from litellm.llms.bedrock.common_utils import (
     BedrockModelInfo,
     bedrock_route_for_request,
@@ -2634,6 +2635,11 @@ def _complete_custom_openai(
     )
 
     headers = headers or litellm.headers
+    outbound_headers: Final = (
+        headers
+        if provider_config is None
+        else with_attribution_headers(provider_config.get_attribution_headers(), headers)
+    )
 
     # Add GitHub Copilot headers (same as /responses endpoint does)
     if custom_llm_provider == "github_copilot":
@@ -2685,7 +2691,7 @@ def _complete_custom_openai(
                 acompletion=acompletion,
                 stream=stream,
                 api_key=api_key,
-                headers=headers,
+                headers=outbound_headers,
                 client=client,
                 provider_config=provider_config,
             )
@@ -2693,7 +2699,7 @@ def _complete_custom_openai(
             response = openai_chat_completions.completion(
                 model=model,
                 messages=messages,
-                headers=headers,
+                headers=outbound_headers,
                 model_response=model_response,
                 print_verbose=print_verbose,
                 api_key=api_key,
@@ -2716,7 +2722,7 @@ def _complete_custom_openai(
             input=messages,
             api_key=api_key,
             original_response=str(e),
-            additional_args={"headers": headers},
+            additional_args={"headers": outbound_headers},
         )
         raise e
 
@@ -2726,7 +2732,7 @@ def _complete_custom_openai(
             input=messages,
             api_key=api_key,
             original_response=response,
-            additional_args={"headers": headers},
+            additional_args={"headers": outbound_headers},
         )
 
     return response  # pyright: ignore[reportReturnType]  # provider SDK return type is broader than the dispatch contract

litellm/model_prices_and_context_window_backup.json

@@ -15229,8 +15229,8 @@
         "cache_creation_input_token_cost": 2.5e-06,
         "cache_creation_input_token_cost_above_1hr": 4e-06,
         "cache_creation_input_token_cost_batches": 1.25e-06,
-        "cache_read_input_token_cost": 2e-07,
-        "cache_read_input_token_cost_batches": 1e-07,
+        "cache_read_input_token_cost": 1e-07,
+        "cache_read_input_token_cost_batches": 5e-08,
         "input_cost_per_token": 2e-06,
         "input_cost_per_token_batches": 1e-06,
         "litellm_provider": "anthropic",
@@ -15267,7 +15267,7 @@
         "supports_forced_tool_use": false,
         "thinking_always_on": true,
         "prompt_cache_min_tokens": 512,
-        "source": "https://platform.claude.com/docs/en/models/sonnet-5-5/overview",
+        "source": "https://platform.claude.com/docs/en/about-claude/pricing",
         "supports_web_search": true
     },
     "claude-sonnet-4-6": {
@@ -80768,5 +80768,648 @@
         "supports_system_messages": true,
         "supports_tool_choice": true,
         "supports_vision": true
+    },
+"claude-haiku-5-5": {
+        "supports_anthropic_compaction": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_creation_input_token_cost_batches": 6.25e-08,
+        "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_batches": 5e-09,
+        "input_cost_per_token": 1e-07,
+        "input_cost_per_token_batches": 5e-08,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "output_cost_per_token_batches": 2.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_native_structured_output": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview",
+        "supports_web_search": true,
+        "input_cost_per_token_above_100k_tokens": 5e-07,
+        "output_cost_per_token_above_100k_tokens": 2.5e-06,
+        "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
+        "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
+        "cache_read_input_token_cost_above_100k_tokens": 5e-08
+    },
+    "bedrock_mantle/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_mantle",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
+    },
+    "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock_mantle",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
+    },
+    "anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
+    },
+    "apac.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "input_cost_per_token": 1.1e-07,
+        "output_cost_per_token": 5.5e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
+    },
+    "au.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "azure_ai/claude-haiku-5-5": {
+        "supports_mid_conversation_system": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "deprecation_date": "2027-09-29",
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_forced_tool_use": true,
+        "thinking_always_on": false,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
+    },
+    "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "eu.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "global.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "jp.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "perplexity/anthropic/claude-haiku-5-5": {
+        "litellm_provider": "perplexity",
+        "mode": "responses",
+        "supports_adaptive_thinking": true,
+        "supports_web_search": true,
+        "supports_function_calling": true,
+        "input_cost_per_token": 1e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "source": "https://docs.perplexity.ai/docs/agent-api/models"
+    },
+    "us-gov.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "us.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "vertex_ai/claude-haiku-5-5": {
+        "regional_endpoint_uplift_multiplier": 1.1,
+        "supports_mid_conversation_system": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_creation_input_token_cost_batches": 6.25e-08,
+        "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_batches": 5e-09,
+        "input_cost_per_token": 1e-07,
+        "input_cost_per_token_batches": 5e-08,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "output_cost_per_token_batches": 2.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_forced_tool_use": true,
+        "thinking_always_on": false,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
+    },
+    "vertex_ai/claude-haiku-5-5@default": {
+        "regional_endpoint_uplift_multiplier": 1.1,
+        "supports_mid_conversation_system": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_creation_input_token_cost_batches": 6.25e-08,
+        "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_batches": 5e-09,
+        "input_cost_per_token": 1e-07,
+        "input_cost_per_token_batches": 5e-08,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "output_cost_per_token_batches": 2.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_forced_tool_use": true,
+        "thinking_always_on": false,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     }
 }

litellm/proxy/_lazy_openapi_snapshot.json

@@ -11572,6 +11572,23 @@
               "description": "Google Cloud location/region (e.g., us-central1)",
               "title": "Location"
             },
+            "logging_only_scope": {
+              "anyOf": [
+                {
+                  "enum": [
+                    "input",
+                    "output",
+                    "both"
+                  ],
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.",
+              "title": "Logging Only Scope"
+            },
             "mask_request_content": {
               "anyOf": [
                 {
@@ -12727,6 +12744,77 @@
           "title": "GuardrailSubmissionSummary",
           "type": "object"
         },
+        "GuardrailUIAddGuardrailSettings": {
+          "properties": {
+            "content_filter_settings": {
+              "anyOf": [
+                {
+                  "additionalProperties": true,
+                  "type": "object"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Content Filter Settings"
+            },
+            "pii_entity_categories": {
+              "items": {
+                "$ref": "#/components/schemas/PiiEntityCategoryMap"
+              },
+              "title": "Pii Entity Categories",
+              "type": "array"
+            },
+            "providers_without_directional_logging_only_scope": {
+              "items": {
+                "type": "string"
+              },
+              "title": "Providers Without Directional Logging Only Scope",
+              "type": "array"
+            },
+            "supported_actions": {
+              "items": {
+                "type": "string"
+              },
+              "title": "Supported Actions",
+              "type": "array"
+            },
+            "supported_entities": {
+              "items": {
+                "type": "string"
+              },
+              "title": "Supported Entities",
+              "type": "array"
+            },
+            "supported_modes": {
+              "items": {
+                "type": "string"
+              },
+              "title": "Supported Modes",
+              "type": "array"
+            },
+            "supported_modes_by_provider": {
+              "additionalProperties": {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              },
+              "title": "Supported Modes By Provider",
+              "type": "object"
+            }
+          },
+          "required": [
+            "supported_entities",
+            "supported_actions",
+            "supported_modes",
+            "supported_modes_by_provider",
+            "providers_without_directional_logging_only_scope",
+            "pii_entity_categories"
+          ],
+          "title": "GuardrailUIAddGuardrailSettings",
+          "type": "object"
+        },
         "HTTPValidationError": {
           "properties": {
             "detail": {
@@ -13822,6 +13910,23 @@
               "description": "Google Cloud location/region (e.g., us-central1)",
               "title": "Location"
             },
+            "logging_only_scope": {
+              "anyOf": [
+                {
+                  "enum": [
+                    "input",
+                    "output",
+                    "both"
+                  ],
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.",
+              "title": "Logging Only Scope"
+            },
             "mask": {
               "anyOf": [
                 {
@@ -14882,6 +14987,27 @@
           "title": "PiiAction",
           "type": "string"
         },
+        "PiiEntityCategoryMap": {
+          "properties": {
+            "category": {
+              "title": "Category",
+              "type": "string"
+            },
+            "entities": {
+              "items": {
+                "type": "string"
+              },
+              "title": "Entities",
+              "type": "array"
+            }
+          },
+          "required": [
+            "category",
+            "entities"
+          ],
+          "title": "PiiEntityCategoryMap",
+          "type": "object"
+        },
         "PiiEntityType": {
           "enum": [
             "CREDIT_CARD",
@@ -16323,7 +16449,9 @@
             "200": {
               "content": {
                 "application/json": {
-                  "schema": {}
+                  "schema": {
+                    "$ref": "#/components/schemas/GuardrailUIAddGuardrailSettings"
+                  }
                 }
               },
               "description": "Successful Response"

litellm/proxy/db/baseline_accounting.py

@@ -223,7 +223,8 @@ ON CONFLICT (request_id) DO NOTHING
 _MARK_CONFLICT: Final = """
 UPDATE "LiteLLM_AutoRouterBaselineObservation"
 SET conflicted = TRUE, revision = $4::bigint
-WHERE request_id = $1 AND scope = $2 AND data <> $3 AND NOT conflicted
+WHERE request_id = $1 AND scope = $2 AND NOT conflicted
+  AND (data::jsonb #- '{turn,turn_at}') <> ($3::jsonb #- '{turn,turn_at}')
 """
 _READ_PAGE: Final = """
 WITH times AS (

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -36,9 +36,11 @@ from litellm.proxy.guardrails.guardrail_hooks.custom_code.sandbox import (
 )
 from litellm.proxy.guardrails.guardrail_registry import (
     GuardrailRegistry,
+    configured_event_hooks,
     contains_encrypted_marker,
     decrypt_guardrail_litellm_params,
     encrypt_guardrail_litellm_params,
+    parse_tolerant_litellm_params,
 )
 from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router
 from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
@@ -283,7 +285,13 @@ async def list_guardrails_v2(
                 number_of_asterisks=4,
             )
             masked_litellm_params = (
-                BaseLitellmParams.model_validate(masked_litellm_params_dict) if masked_litellm_params_dict else None
+                parse_tolerant_litellm_params(
+                    masked_litellm_params_dict,
+                    guardrail.get("guardrail_name") or "Unknown",
+                    params_model=BaseLitellmParams,
+                )
+                if masked_litellm_params_dict
+                else None
             )
             guardrail_configs.append(
                 GuardrailInfoResponse(
@@ -324,7 +332,11 @@ async def list_guardrails_v2(
                 number_of_asterisks=4,
             )
             masked_in_memory_litellm_params_typed = (
-                BaseLitellmParams.model_validate(masked_in_memory_litellm_params)
+                parse_tolerant_litellm_params(
+                    masked_in_memory_litellm_params,
+                    guardrail.get("guardrail_name") or "Unknown",
+                    params_model=BaseLitellmParams,
+                )
                 if masked_in_memory_litellm_params
                 else None
             )
@@ -425,7 +437,11 @@ async def create_guardrail(
         guardrail_id: Final = result.get("guardrail_id", "Unknown")
 
         try:
-            IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(guardrail=cast(Guardrail, result), source="db")
+            IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
+                guardrail=cast(Guardrail, result),
+                source="db",
+                reject_invalid_logging_only_scope=True,
+            )
             verbose_proxy_logger.info(
                 "Immediate sync: Successfully initialized guardrail '%s' (ID: %s)", guardrail_name, guardrail_id
             )
@@ -550,7 +566,10 @@ async def update_guardrail(
         guardrail_name: Final = result.get("guardrail_name", "Unknown")
 
         try:
-            IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=cast(Guardrail, result))
+            IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
+                guardrail=cast(Guardrail, result),
+                reject_invalid_logging_only_scope=True,
+            )
             verbose_proxy_logger.info(
                 "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id
             )
@@ -1240,19 +1259,35 @@ async def patch_guardrail(
 
         # Update litellm_params if default_on is provided or pii_entities_config is provided
         existing_litellm_params: Final = _as_str_object_mapping(dict(existing_guardrail.get("litellm_params", {})))
-        litellm_params = LitellmParams(**existing_litellm_params)
-        if request.litellm_params is not None:
-            requested_litellm_params: Final = request.litellm_params.model_dump(exclude_unset=True)
-            litellm_params_dict: Final = litellm_params.model_dump(exclude_unset=True)
-            litellm_params_dict.update(requested_litellm_params)
-            merged_litellm_params: Final = _as_str_object_mapping(litellm_params_dict)
-            try:
-                litellm_params = LitellmParams(**merged_litellm_params)
-            except ValidationError as validation_error:
-                raise HTTPException(
-                    status_code=422,
-                    detail=f"Invalid guardrail configuration, update rejected: {validation_error}",
-                ) from validation_error
+        current_litellm_params: Final = parse_tolerant_litellm_params(
+            existing_litellm_params,
+            existing_guardrail.get("guardrail_name") or "Unknown",
+        )
+        requested_litellm_params: Final[Mapping[str, object]] = (
+            MappingProxyType(request.litellm_params.model_dump(exclude_unset=True))
+            if request.litellm_params is not None
+            else MappingProxyType({})
+        )
+        merged_litellm_params: Final = _as_str_object_mapping(
+            MappingProxyType({**current_litellm_params.model_dump(exclude_unset=True), **requested_litellm_params})
+        )
+        try:
+            parsed_litellm_params: Final = LitellmParams(**merged_litellm_params)
+        except ValidationError as validation_error:
+            raise HTTPException(
+                status_code=422,
+                detail=f"Invalid guardrail configuration, update rejected: {validation_error}",
+            ) from validation_error
+        clear_stored_scope: Final = (
+            "logging_only_scope" not in requested_litellm_params
+            and parsed_litellm_params.logging_only_scope is not None
+            and GuardrailEventHooks.logging_only.value not in configured_event_hooks(parsed_litellm_params.mode)
+        )
+        litellm_params: Final = (
+            LitellmParams(**MappingProxyType({**merged_litellm_params, "logging_only_scope": None}))
+            if clear_stored_scope
+            else parsed_litellm_params
+        )
 
         # Update guardrail_info if provided
         guardrail_info: Final = (
@@ -1281,6 +1316,7 @@ async def patch_guardrail(
         try:
             IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
                 guardrail=guardrail,
+                reject_invalid_logging_only_scope="logging_only_scope" in requested_litellm_params,
             )
             verbose_proxy_logger.info(
                 "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id
@@ -1294,15 +1330,7 @@ async def patch_guardrail(
             # the caller instead of a misleading 200.
             await GUARDRAIL_REGISTRY.update_guardrail_in_db(
                 guardrail_id=guardrail_id,
-                guardrail=Guardrail(
-                    guardrail_id=guardrail_id,
-                    guardrail_name=existing_guardrail.get("guardrail_name") or "",
-                    litellm_params=LitellmParams(**existing_litellm_params),
-                    guardrail_info=existing_guardrail.get(
-                        "guardrail_info",
-                        {},
-                    ),
-                ),
+                guardrail=existing_guardrail,
                 prisma_client=prisma_client,
             )
             raise HTTPException(
@@ -1404,7 +1432,13 @@ async def get_guardrail_info(guardrail_id: str):
             number_of_asterisks=4,
         )
         masked_litellm_params = (
-            BaseLitellmParams.model_validate(masked_litellm_params_dict) if masked_litellm_params_dict else None
+            parse_tolerant_litellm_params(
+                masked_litellm_params_dict,
+                result.get("guardrail_name") or "Unknown",
+                params_model=BaseLitellmParams,
+            )
+            if masked_litellm_params_dict
+            else None
         )
 
         return GuardrailInfoResponse(
@@ -1427,7 +1461,7 @@ async def get_guardrail_info(guardrail_id: str):
     tags=["Guardrails"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def get_guardrail_ui_settings():
+async def get_guardrail_ui_settings() -> GuardrailUIAddGuardrailSettings:
     """
     Get the UI settings for the guardrails
 
@@ -1461,12 +1495,18 @@ async def get_guardrail_ui_settings():
         # above; it only runs on pre_call.
         {SupportedGuardrailIntegrations.HIDE_SECRETS.value: [GuardrailEventHooks.pre_call.value]}
     )
+    providers_without_directional_logging_only_scope: Final = tuple(
+        provider
+        for provider, guardrail_class in guardrail_class_registry.items()
+        if not guardrail_class.supports_logging_only_scope()
+    )
 
     return GuardrailUIAddGuardrailSettings(
         supported_entities=[entity.value for entity in PiiEntityType],
         supported_actions=[action.value for action in PiiAction],
         supported_modes=[mode.value for mode in GuardrailEventHooks],
         supported_modes_by_provider=supported_modes_by_provider,
+        providers_without_directional_logging_only_scope=providers_without_directional_logging_only_scope,
         pii_entity_categories=category_maps,
         content_filter_settings={
             "prebuilt_patterns": get_pattern_metadata(),

litellm/proxy/guardrails/guardrail_initializers.py

@@ -100,7 +100,7 @@ _MCP_EVENT_HOOKS: Final = frozenset(
 )
 
 
-def _configured_event_hooks(mode: str | list[str] | Mode) -> tuple[str, ...]:
+def configured_event_hooks(mode: str | list[str] | Mode) -> tuple[str, ...]:
     if isinstance(mode, str):
         return (mode,)
     if isinstance(mode, list):
@@ -114,7 +114,7 @@ def _configured_event_hooks(mode: str | list[str] | Mode) -> tuple[str, ...]:
 
 
 def _is_mcp_only_mode(mode: str | list[str] | Mode) -> bool:
-    hooks: Final = _configured_event_hooks(mode)
+    hooks: Final = configured_event_hooks(mode)
     return bool(hooks) and all(hook in _MCP_EVENT_HOOKS for hook in hooks)
 
 

litellm/proxy/guardrails/guardrail_registry.py

@@ -6,7 +6,8 @@ import os
 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
 from datetime import datetime, timezone
 from itertools import chain, count
-from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeAlias, cast
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeAlias, TypeVar, cast
 
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
@@ -58,6 +59,7 @@ from .guardrail_hooks.llm_as_a_judge import (
     initialize_guardrail as initialize_llm_as_a_judge,
 )
 from .guardrail_initializers import (
+    configured_event_hooks,
     initialize_bedrock,
     initialize_hide_secrets,
     initialize_lakera,
@@ -572,9 +574,46 @@ def _as_callback_tuple(
     return (initialized,)
 
 
-def _configure_callback_scoping(
+def _logging_only_scope_error(
     custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams
+) -> str | None:
+    logging_only_scope: Final = litellm_params.logging_only_scope
+    if logging_only_scope is not None and GuardrailEventHooks.logging_only.value not in configured_event_hooks(
+        litellm_params.mode
+    ):
+        return (
+            f"Guardrail {guardrail_name}: logging_only_scope is set, but mode does not include logging_only, "
+            "so it would never apply. Add logging_only to mode or remove logging_only_scope."
+        )
+    if logging_only_scope in ("input", "output") and not custom_guardrail_callback.supports_logging_only_scope():
+        return (
+            f"Guardrail {guardrail_name}: logging_only_scope={logging_only_scope!r} is not supported by this "
+            "guardrail, whose logging_only hook scans on its own. Remove logging_only_scope."
+        )
+    return None
+
+
+def _configure_callback_scoping(
+    custom_guardrail_callback: CustomGuardrail,
+    guardrail_name: str,
+    litellm_params: LitellmParams,
+    *,
+    reject_invalid_logging_only_scope: bool = False,
 ) -> None:
+    logging_only_scope: Final = litellm_params.logging_only_scope
+    logging_only_scope_error: Final = _logging_only_scope_error(
+        custom_guardrail_callback, guardrail_name, litellm_params
+    )
+    if logging_only_scope_error is not None:
+        if reject_invalid_logging_only_scope:
+            raise ValueError(logging_only_scope_error)
+        verbose_proxy_logger.error(
+            "%s Ignoring logging_only_scope; the guardrail keeps its configured mode.",
+            logging_only_scope_error.replace("\r", "").replace("\n", ""),
+        )
+        custom_guardrail_callback.logging_only_scope = None
+    else:
+        custom_guardrail_callback.logging_only_scope = logging_only_scope
     for scoping_param in (
         "skip_system_message_in_guardrail",
         "skip_tool_message_in_guardrail",
@@ -597,6 +636,28 @@ def _configure_callback_scoping(
     _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params)
 
 
+_ParamsT = TypeVar("_ParamsT", bound=BaseModel)
+
+
+def parse_tolerant_litellm_params(
+    litellm_params_data: Mapping[str, object],
+    guardrail_name: str,
+    params_model: type[_ParamsT] = LitellmParams,
+) -> _ParamsT:
+    try:
+        return params_model(**litellm_params_data)
+    except ValidationError as validation_error:
+        if any(tuple(error["loc"]) != ("logging_only_scope",) for error in validation_error.errors()):
+            raise
+        verbose_proxy_logger.error(
+            "Guardrail %s: logging_only_scope=%r is not one of 'input', 'output' or 'both'. "
+            "Ignoring logging_only_scope; the guardrail keeps its configured mode.",
+            guardrail_name.replace("\r", "").replace("\n", ""),
+            str(litellm_params_data.get("logging_only_scope")).replace("\r", "").replace("\n", "")[:100],
+        )
+        return params_model(**MappingProxyType({**litellm_params_data, "logging_only_scope": None}))
+
+
 class InMemoryGuardrailHandler:
     """
     Class that handles initializing guardrails and adding them to the CallbackManager
@@ -633,6 +694,8 @@ class InMemoryGuardrailHandler:
         config_file_path: str | None = None,
         llm_router: Optional["Router"] = None,
         source: Literal["db", "config"] = "config",
+        *,
+        reject_invalid_logging_only_scope: bool = False,
     ) -> Guardrail | None:
         """
         Initialize a guardrail from a dictionary and add it to the litellm callback manager
@@ -653,7 +716,10 @@ class InMemoryGuardrailHandler:
         verbose_proxy_logger.debug("litellm_params= %s", litellm_params_data)
 
         if isinstance(litellm_params_data, dict):
-            litellm_params = LitellmParams(**litellm_params_data)
+            if reject_invalid_logging_only_scope:
+                litellm_params = LitellmParams(**litellm_params_data)
+            else:
+                litellm_params = parse_tolerant_litellm_params(litellm_params_data, guardrail["guardrail_name"])
         else:
             litellm_params = litellm_params_data
 
@@ -679,8 +745,18 @@ class InMemoryGuardrailHandler:
             config_file_path=config_file_path,
             llm_router=llm_router,
         )
-        for custom_guardrail_callback in created_callbacks:
-            _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params)
+        try:
+            for custom_guardrail_callback in created_callbacks:
+                _configure_callback_scoping(
+                    custom_guardrail_callback,
+                    guardrail["guardrail_name"],
+                    litellm_params,
+                    reject_invalid_logging_only_scope=reject_invalid_logging_only_scope,
+                )
+        except Exception:
+            for custom_guardrail_callback in created_callbacks:
+                litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback)
+            raise
 
         parsed_guardrail: Final = Guardrail(
             guardrail_id=guardrail.get("guardrail_id"),
@@ -729,6 +805,28 @@ class InMemoryGuardrailHandler:
         siblings: Final = self.guardrail_id_to_sibling_callbacks.get(guardrail_id, ())
         return (() if primary is None else (primary,)) + siblings
 
+    def _reject_invalid_logging_only_scope(self, guardrail_id: str, guardrail: Guardrail) -> None:
+        """
+        Strictly validate logging_only_scope on a row whose params are otherwise
+        unchanged, without rebuilding the live callback.
+
+        API write paths send the whole object, so an invalid scope must still be
+        rejected even when the write changed nothing else. But an unchanged row
+        must not force a teardown + re-append: initialize_guardrail appends the
+        rebuilt callback at the END of litellm.callbacks, so a no-op PUT would
+        reorder guardrails and change which one wins between a BLOCK and a MASK
+        guardrail over the same content.
+        """
+        params: Final = guardrail.get("litellm_params")
+        if not isinstance(params, (dict, LitellmParams)):
+            return
+        litellm_params: Final = LitellmParams(**params) if isinstance(params, dict) else params
+        guardrail_name: Final = guardrail.get("guardrail_name", "Unknown")
+        for custom_guardrail_callback in self._tracked_callbacks(guardrail_id):
+            scope_error = _logging_only_scope_error(custom_guardrail_callback, guardrail_name, litellm_params)
+            if scope_error is not None:
+                raise ValueError(scope_error)
+
     def initialize_custom_guardrail(
         self,
         guardrail: Guardrail,
@@ -788,6 +886,8 @@ class InMemoryGuardrailHandler:
         guardrail_id: str,
         guardrail: Guardrail,
         source: Literal["db", "config"] = "db",
+        *,
+        reject_invalid_logging_only_scope: bool = False,
     ) -> None:
         """
         Update a guardrail in memory: a changed name or litellm_params rebuilds the
@@ -796,8 +896,14 @@ class InMemoryGuardrailHandler:
         """
         updated_guardrail: Final = cast(Guardrail, {**guardrail, "guardrail_id": guardrail_id})
         if self._has_guardrail_params_changed(guardrail_id, updated_guardrail):
-            self.reinitialize_guardrail(guardrail=updated_guardrail, source=source)
+            self.reinitialize_guardrail(
+                guardrail=updated_guardrail,
+                source=source,
+                reject_invalid_logging_only_scope=reject_invalid_logging_only_scope,
+            )
             return
+        if reject_invalid_logging_only_scope:
+            self._reject_invalid_logging_only_scope(guardrail_id, updated_guardrail)
         self.IN_MEMORY_GUARDRAILS[guardrail_id] = updated_guardrail
         self._sources[guardrail_id] = source
 
@@ -883,6 +989,7 @@ class InMemoryGuardrailHandler:
     @staticmethod
     def _normalize_litellm_params_for_comparison(
         params: LitellmParams | Mapping[str, object] | None,
+        guardrail_name: str,
     ) -> Mapping[str, object] | None:
         """
         Render litellm_params to a canonical dict so an in-memory LitellmParams and
@@ -899,7 +1006,7 @@ class InMemoryGuardrailHandler:
             return params.model_dump()
         if isinstance(params, dict):
             try:
-                return LitellmParams(**params).model_dump()
+                return parse_tolerant_litellm_params(params, guardrail_name).model_dump()
             except ValidationError as e:
                 verbose_proxy_logger.warning(
                     "Could not normalize guardrail litellm_params for comparison; treating the guardrail as changed. Error: %s",
@@ -922,8 +1029,12 @@ class InMemoryGuardrailHandler:
             return True
 
         # Compare litellm_params
-        existing_dict: Final = self._normalize_litellm_params_for_comparison(existing.get("litellm_params"))
-        new_dict: Final = self._normalize_litellm_params_for_comparison(new_guardrail.get("litellm_params"))
+        existing_dict: Final = self._normalize_litellm_params_for_comparison(
+            existing.get("litellm_params"), existing.get("guardrail_name", "Unknown")
+        )
+        new_dict: Final = self._normalize_litellm_params_for_comparison(
+            new_guardrail.get("litellm_params"), new_guardrail.get("guardrail_name", "Unknown")
+        )
 
         # Compare and identify specific differences
         changed_fields = {}
@@ -949,6 +1060,8 @@ class InMemoryGuardrailHandler:
         guardrail: Guardrail,
         config_file_path: str | None = None,
         source: Literal["db", "config"] = "config",
+        *,
+        reject_invalid_logging_only_scope: bool = False,
     ) -> Guardrail | None:
         """
         Force re-initialization of a guardrail even if it exists in memory.
@@ -978,7 +1091,12 @@ class InMemoryGuardrailHandler:
         # instance instead of leaving the guardrail silently removed: a guardrail
         # that was enforcing must never fail open because an update was bad.
         try:
-            return self.initialize_guardrail(guardrail=guardrail, config_file_path=config_file_path, source=source)
+            return self.initialize_guardrail(
+                guardrail=guardrail,
+                config_file_path=config_file_path,
+                source=source,
+                reject_invalid_logging_only_scope=reject_invalid_logging_only_scope,
+            )
         except Exception as init_error:
             if previous_guardrail is not None:
                 verbose_proxy_logger.exception(
@@ -1003,7 +1121,9 @@ class InMemoryGuardrailHandler:
         )
         if existing is None or db_params is None or not contains_encrypted_marker(db_params):
             return guardrail
-        loaded_params: Final = self._normalize_litellm_params_for_comparison(existing.get("litellm_params"))
+        loaded_params: Final = self._normalize_litellm_params_for_comparison(
+            existing.get("litellm_params"), guardrail.get("guardrail_name", "Unknown")
+        )
         verbose_proxy_logger.warning(
             "Guardrail %s has litellm_params that do not decrypt with the current key; keeping the loaded values for "
             "them. Restart the proxy if the master key was rotated.",
@@ -1023,7 +1143,13 @@ class InMemoryGuardrailHandler:
             }
         )
 
-    def sync_guardrail_from_db(self, guardrail: Guardrail, config_file_path: str | None = None) -> Guardrail | None:
+    def sync_guardrail_from_db(
+        self,
+        guardrail: Guardrail,
+        config_file_path: str | None = None,
+        *,
+        reject_invalid_logging_only_scope: bool = False,
+    ) -> Guardrail | None:
         """
         Sync a guardrail from DB - initializes if new, re-initializes if changed.
         DB values that do not decrypt with the current key keep the loaded guardrail's values.
@@ -1044,8 +1170,12 @@ class InMemoryGuardrailHandler:
                 guardrail=synced,
                 config_file_path=config_file_path,
                 source="db",
+                reject_invalid_logging_only_scope=reject_invalid_logging_only_scope,
             )
 
+        if reject_invalid_logging_only_scope:
+            self._reject_invalid_logging_only_scope(guardrail_id, synced)
+
         # Params unchanged but the entry is still DB-backed; make sure the
         # source marker reflects that even if it was previously set differently
         # (e.g. a config entry whose UUID later collided with a DB row).

litellm/proxy/hooks/autorouter_baseline_cache.py

@@ -5,7 +5,7 @@ import hashlib
 import json
 import time
 from collections.abc import Callable, Mapping
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
 from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
@@ -16,9 +16,8 @@ from pydantic import ConfigDict, Field, JsonValue, TypeAdapter
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.core_helpers import (
-    get_litellm_metadata_from_kwargs,  # pyright: ignore[reportUnknownVariableType]  # legacy metadata boundary validated below
-)
+from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+from litellm.litellm_core_utils.logging_worker import optional_callback_budget
 from litellm.llms.anthropic.prompt_cache_prediction import (
     CountedPromptCachePlan,
     NativePredictionTarget,
@@ -28,6 +27,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import (
     count_cache_plan,
     count_prompt_tokens,
     parse_cache_plan,
+    prepare_native_baseline_body,
     resolve_baseline_prediction_target,
     supported_baseline_recipient,
     supported_prediction_headers,
@@ -37,6 +37,8 @@ from litellm.proxy.spend_tracking.savings import (
     _effective_model_info,  # pyright: ignore[reportPrivateUsage]  # existing deployment-price owner
     _proxy_llm_router,  # pyright: ignore[reportPrivateUsage]  # existing optional proxy-router owner
 )
+from litellm.router_strategy.complexity_router.context_compaction import compaction_applied
+from litellm.router_utils.baseline_request import baseline_request
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import BaselineRouteStamp
 from litellm.types.utils import CallTypes, ModelInfo, Usage
@@ -66,6 +68,9 @@ class CapturedBaselineObservation(LiteLLMBaseModel):
     prices: ModelInfo | None
     observation: BaselineObservation
 
+    def with_observation(self, observation: BaselineObservation) -> CapturedBaselineObservation:
+        return self.model_copy(update={"observation": observation})
+
 
 @dataclass(frozen=True, slots=True)
 class BaselineCacheContext:
@@ -73,7 +78,10 @@ class BaselineCacheContext:
     capture: CapturedBaselineObservation
     target: NativePredictionTarget | UnsupportedPredictionTarget
     baseline_deployment_id: str
+    baseline_body: Mapping[str, JsonValue] | None = field(default=None, repr=False)
+    selected_body_digest: str | None = field(default=None, repr=False)
     invalidated: str | None = None
+    finalization: asyncio.Task[CapturedBaselineObservation] | None = field(default=None, repr=False, compare=False)
 
 
 class _Metadata(LiteLLMBaseModel):
@@ -102,6 +110,10 @@ def _digest(value: object) -> str:
     return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
 
 
+def _native_body_digest(body: Mapping[str, JsonValue]) -> str:
+    return _digest({key: value for key, value in body.items() if key not in ("metadata", "stream")})
+
+
 class AutoRouterBaselineCache(CustomLogger):
     def __init__(
         self,
@@ -124,12 +136,15 @@ class AutoRouterBaselineCache(CustomLogger):
         if not isinstance(logging_obj, Logging) or call_type != CallTypes.anthropic_messages:
             return
         try:
-            metadata: Final = _METADATA.validate_python(get_litellm_metadata_from_kwargs({"litellm_params": kwargs}))
+            raw_metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs))
+            metadata: Final = _METADATA.validate_python(raw_metadata) if isinstance(raw_metadata, Mapping) else {}
             if metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY):
                 return
             if logging_obj.baseline_cache_context is not None:
                 await invalidate_baseline_cache(logging_obj, "retried_request")
                 return
+            if not isinstance(metadata.get("_autorouter_baseline_route"), BaselineRouteStamp):
+                return
             request: Final = _Metadata.model_validate(metadata)
             session: Final = kwargs.get("litellm_session_id") or request.session_id or logging_obj.litellm_session_id
             if not isinstance(session, str) or not session or len(session) > 256:
@@ -142,13 +157,27 @@ class AutoRouterBaselineCache(CustomLogger):
             prices: Final = _PRICES.validate_python(
                 _effective_model_info(router, request.route.baseline_deployment_id, request.route.baseline_model)
             )
+            params: Final = (
+                _METADATA.validate_python(deployment.litellm_params.model_dump(mode="json")) if deployment else {}
+            )
+            projected: Final = (
+                baseline_request(
+                    kwargs,
+                    request.route.request_parameters,
+                    params,
+                    include_extra_body=False,
+                )
+                if request.route.request_parameters is not None
+                else None
+            )
             scope: Final = "autorouter-baseline:v3:" + _digest(
                 (
+                    "baseline_request_v4",
                     request.user_api_key_hash,
                     session,
                     request.route.router_name,
                     request.route.baseline_deployment_id,
-                    deployment.litellm_params.model_dump(mode="json"),
+                    params,
                     prices,
                 )
             )
@@ -170,8 +199,21 @@ class AutoRouterBaselineCache(CustomLogger):
                     reason="incomplete_response",
                 ),
             )
+            selected_model: Final = kwargs.get("model")
+            selected_body: Final = prepare_native_baseline_body(
+                kwargs, selected_model if isinstance(selected_model, str) else logging_obj.model
+            )
             logging_obj.baseline_cache_context = BaselineCacheContext(
-                self, capture, target, request.route.baseline_deployment_id
+                self,
+                capture,
+                target,
+                request.route.baseline_deployment_id,
+                prepare_native_baseline_body(projected, target.model)
+                if projected is not None and isinstance(target, NativePredictionTarget)
+                else None,
+                _native_body_digest(selected_body)
+                if selected_body is not None and not compaction_applied(kwargs)
+                else None,
             )
         except Exception:  # noqa: BLE001  # optional observation cannot fail inference
             verbose_proxy_logger.warning("Auto-router baseline observation could not be initialized")
@@ -197,14 +239,16 @@ class AutoRouterBaselineCache(CustomLogger):
     async def plan(
         self, target: NativePredictionTarget, wire: httpx.Request, body: Mapping[str, JsonValue], usage: Usage | None
     ) -> tuple[CountedPromptCachePlan | None, str | None]:
+        deadline: Final = asyncio.get_running_loop().time() + optional_callback_budget(_COUNT_TIMEOUT, fraction=0.75)
         if not supported_prediction_headers(wire.headers):
             return None, "unsupported_request_headers"
         plan: Final = parse_cache_plan(body)
         if isinstance(plan, UnsupportedCachePlan):
             return None, plan.reason
         details: Final = usage.prompt_tokens_details if usage is not None else None
+        selected: Final = parse_cache_plan(_JSON_BODY.validate_json(wire.content))
         if (
-            not plan.breakpoints
+            (isinstance(selected, UnsupportedCachePlan) or not selected.breakpoints)
             and details is not None
             and ((details.cached_tokens or 0) + (details.cache_creation_tokens or 0))
         ):
@@ -215,7 +259,8 @@ class AutoRouterBaselineCache(CustomLogger):
 
         try:
             counted: Final = await asyncio.wait_for(
-                count_cache_plan(target.model, target.api_key, plan, token_counter=count), timeout=_COUNT_TIMEOUT
+                count_cache_plan(target.model, target.api_key, plan, token_counter=count),
+                timeout=max(0.0, deadline - asyncio.get_running_loop().time()),
             )
             return (None, counted.reason) if isinstance(counted, UnsupportedCachePlan) else (counted, None)
         except TimeoutError:
@@ -229,111 +274,121 @@ async def invalidate_baseline_cache(logging_obj: Logging, reason: str, *, comple
     if context is not None:
         logging_obj.baseline_cache_context = replace(context, invalidated=reason)
         logging_obj.baseline_observation = context.capture.model_copy(
-            update=MappingProxyType(
-                {
-                    "observation": context.capture.observation.model_copy(
-                        update=MappingProxyType(
-                            {
-                                "available_at": max(context.capture.observation.started_at, context.collector.clock()),
-                                "reason": reason,
-                            }
-                        )
-                    ),
-                }
-            )
+            update={
+                "observation": context.capture.observation.model_copy(
+                    update={
+                        "available_at": max(context.capture.observation.started_at, context.collector.clock()),
+                        "reason": reason,
+                    }
+                ),
+            }
         )
 
 
 async def finalize_baseline_cache(logging_obj: Logging, response_obj: object) -> None:
     context: Final = logging_obj.baseline_cache_context
-    if context is None:
+    if context is None or logging_obj.baseline_observation is not None:
         return
+    task: Final = context.finalization or asyncio.create_task(_capture(context, logging_obj, response_obj))
+    active: Final = context if context.finalization is not None else replace(context, finalization=task)
+    if context.finalization is None:
+        task.add_done_callback(_consume_finalization)
+    logging_obj.baseline_cache_context = active
     try:
-        capture: Final = await _capture(context, logging_obj, response_obj)
-        if logging_obj.baseline_cache_context is context:
-            logging_obj.baseline_observation = capture  # rebind-ok: attach only to the captured request owner
-    except Exception:  # noqa: BLE001  # observation failures must preserve inference and billing
+        capture: Final = await asyncio.shield(task)
+        if logging_obj.baseline_cache_context is active:
+            logging_obj.baseline_observation = capture  # rebind-ok: publish only for the current attempt
+    except Exception:  # noqa: BLE001  # estimation must preserve inference and billing
         await invalidate_baseline_cache(logging_obj, "observation_unavailable")
 
 
-async def _capture(
+def _consume_finalization(task: asyncio.Task[CapturedBaselineObservation]) -> None:
+    if not task.cancelled():
+        task.exception()
+
+
+async def _capture_native(
     context: BaselineCacheContext, logging_obj: Logging, response_obj: object
 ) -> CapturedBaselineObservation:
-    original: Final = context.capture.observation
-    details: Final = _METADATA.validate_python(logging_obj.model_call_details)
-    if details.get("cache_hit") is True:
-        return context.capture.model_copy(
-            update=MappingProxyType(
-                {
-                    "observation": original.model_copy(
-                        update=MappingProxyType({"outcome": "response_cache", "reason": "response_cache_hit"})
-                    )
-                }
-            )
-        )
-    event: Final = _WireEvent.model_validate(details)
+    capture: Final = context.capture
+    original: Final = capture.observation
+    event: Final = _WireEvent.model_validate(logging_obj.model_call_details)
     wire: Final = event.httpx_response.request
     usage: Final = _ResponseUsage.model_validate(response_obj).usage
+    available: Final = event.completion_start_time.timestamp()
     complete: Final = (
         event.custom_llm_provider == "anthropic"
         and event.httpx_response.status_code == 200
         and (not event.stream or event.prompt_cache_response_complete)
     )
-    started: Final = original.started_at
-    available: Final = event.completion_start_time.timestamp()
-    if context.invalidated or not complete or not started <= available <= context.collector.clock():
-        return context.capture.model_copy(
-            update=MappingProxyType(
-                {
-                    "observation": original.model_copy(
-                        update=MappingProxyType(
-                            {
-                                "available_at": max(started, context.collector.clock()),
-                                "reason": context.invalidated or "incomplete_response",
-                            }
-                        )
-                    )
+    if context.invalidated or not complete or not original.started_at <= available <= context.collector.clock():
+        return capture.with_observation(
+            original.model_copy(
+                update={
+                    "available_at": max(original.started_at, context.collector.clock()),
+                    "reason": context.invalidated or "incomplete_response",
                 }
             )
         )
     target: Final = context.target
     if isinstance(target, UnsupportedPredictionTarget) or not supported_baseline_recipient(target, wire):
-        return context.capture.model_copy(
-            update=MappingProxyType(
-                {
-                    "observation": original.model_copy(
-                        update=MappingProxyType(
-                            {
-                                "available_at": available,
-                                "reason": target.reason
-                                if isinstance(target, UnsupportedPredictionTarget)
-                                else "unsupported_baseline_recipient",
-                            }
-                        )
-                    )
+        return capture.with_observation(
+            original.model_copy(
+                update={
+                    "available_at": available,
+                    "reason": target.reason
+                    if isinstance(target, UnsupportedPredictionTarget)
+                    else "unsupported_baseline_recipient",
                 }
             )
         )
     body: Final = _JSON_BODY.validate_json(wire.content)
-    same: Final = (
-        logging_obj.get_router_model_id() == context.baseline_deployment_id and body.get("model") == target.model
-    )
-    plan, reason = await context.collector.plan(target, wire, body, usage)
-    minimum: Final = get_prompt_cache_min_tokens(target.model)
-    return context.capture.model_copy(
-        update=MappingProxyType(
-            {
-                "observation": BaselineObservation(
-                    request_id=original.request_id,
-                    started_at=started,
-                    available_at=available,
-                    outcome="complete",
-                    baseline_equivalent=same,
-                    usage=usage,
-                    plan=plan,
-                    minimum_cache_tokens=minimum,
-                    reason=reason,
-                )
-            }
+    projected: Final = context.baseline_body
+    if projected is None or context.selected_body_digest != _native_body_digest(body):
+        return capture.with_observation(
+            original.model_copy(
+                update={
+                    "available_at": available,
+                    "usage": usage,
+                    "reason": "unsupported_baseline_settings"
+                    if projected is None
+                    else "unsupported_request_transformation",
+                }
+            )
+        )
+    same: Final = logging_obj.get_router_model_id() == context.baseline_deployment_id and _native_body_digest(
+        projected
+    ) == _native_body_digest(body)
+    plan, reason = await context.collector.plan(target, wire, projected, usage)
+    return capture.with_observation(
+        BaselineObservation(
+            request_id=original.request_id,
+            started_at=original.started_at,
+            available_at=available,
+            outcome="complete",
+            baseline_equivalent=same,
+            usage=usage.model_copy(update={key: projected.get(key) for key in ("speed", "inference_geo")})
+            if usage is not None and not same
+            else usage,
+            plan=plan,
+            reason=reason,
+            minimum_cache_tokens=get_prompt_cache_min_tokens(target.model),
         )
     )
+
+
+async def _capture(
+    context: BaselineCacheContext, logging_obj: Logging, response_obj: object
+) -> CapturedBaselineObservation:
+    if _METADATA.validate_python(logging_obj.model_call_details).get("cache_hit") is True:
+        return context.capture.model_copy(
+            update={
+                "observation": context.capture.observation.model_copy(
+                    update={
+                        "outcome": "response_cache",
+                        "reason": "response_cache_hit",
+                    }
+                ),
+            }
+        )
+    return await _capture_native(context, logging_obj, response_obj)

litellm/proxy/lens/endpoints.py

@@ -1,10 +1,11 @@
 import hashlib
 import secrets
+from collections.abc import Awaitable, Callable
 from datetime import datetime, timedelta, timezone
 from functools import reduce
 from itertools import chain
 from types import MappingProxyType
-from typing import Annotated, Final, TypeAlias
+from typing import Annotated, Final, Protocol, TypeAlias
 from uuid import uuid4
 
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
@@ -49,8 +50,10 @@ from litellm.proxy.lens.models import (
     WorkerCreated,
 )
 from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_image
-from litellm.proxy.lens.repository import LensRepository, WriterDatabase
+from litellm.proxy.lens.repository import DueLens, LensRepository, WriterDatabase
 from litellm.proxy.lens.reviews import criteria_key
+from litellm.proxy.lens.signal_repository import SignalRepository
+from litellm.proxy.lens.signals import SignalConfig, TraceSignals, trace_signals
 from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
 from litellm.proxy.lens.state import (
     can_access,
@@ -68,12 +71,29 @@ from litellm.proxy.lens.state import (
     summarized,
 )
 from litellm.proxy.tracing_runtime import provide_storage
+from litellm.router import Router
 from litellm.types.llms.base import LiteLLMBaseModel
 
 router: Final = APIRouter(prefix="/lens", tags=["Lens"])
+CLAIM_CANDIDATES: Final = 20
 _bearer: Final = HTTPBearer()
 Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]
 StorageDep: TypeAlias = Annotated[Storage | None, Depends(provide_storage)]
+SAMPLE_PAGE_SIZE: Final = 10_000
+SAMPLE_PAGE_SIZES: Final = (SAMPLE_PAGE_SIZE, 5_000, 2_500, 1_250, 625, 312, 156, 100)
+SAMPLE_RESPONSE_TOO_LARGE: Final = "ClickHouse query exceeded the response size limit"
+
+
+class _ClaimRepository(Protocol):
+    async def due(
+        self, scope: Scope, now: datetime, limit: int, after: DueLens | None = None
+    ) -> tuple[DueLens, ...]: ...
+
+    async def sync_due(self, lens: Lens) -> None: ...
+
+    async def update(
+        self, lens_id: str, transform: Callable[[Lens], Lens], attempts: int, *, changed_only: bool
+    ) -> Lens | None: ...
 
 
 def repository() -> LensRepository:
@@ -84,6 +104,14 @@ def repository() -> LensRepository:
     return LensRepository(WriterDatabase(writer_wrapper(prisma_client.db)))
 
 
+def signals_repository() -> SignalRepository:
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(503, "Lens needs a connected Postgres database")
+    return SignalRepository(WriterDatabase(writer_wrapper(prisma_client.db)))
+
+
 def source_reader(storage: Storage | None) -> SourceReader:
     if storage is None:
         raise HTTPException(
@@ -101,6 +129,20 @@ def user_scope(auth: UserAPIKeyAuth, write: bool = False) -> Scope:
     raise HTTPException(403, "Lens requires proxy administrator access")
 
 
+def validate_signal_model(config: SignalConfig, llm_router: Router | None) -> None:
+    if not config.model:
+        return
+    message: Final = "Choose a System 1 model (evaluation mode) configured on this proxy"
+    if llm_router is None:
+        raise HTTPException(400, message)
+    try:
+        model_group: Final = llm_router.get_model_group_info(model_group=config.model)
+    except Exception as error:
+        raise HTTPException(400, message) from error
+    if model_group is None or model_group.mode != "evaluation":
+        raise HTTPException(400, message)
+
+
 async def get_lens(lens_id: str, scope: Scope) -> Lens:
     lens: Final = await repository().get(lens_id)
     if lens is None or not can_access(scope, lens.scope):
@@ -244,6 +286,39 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]:
     return await source_reader(storage).agents(scope) if storage is not None else ()
 
 
+@router.get("/signals", response_model=SignalConfig)
+async def get_signals(auth: Auth) -> SignalConfig:
+    user_scope(auth)
+    return await signals_repository().get_config()
+
+
+@router.put("/signals", response_model=SignalConfig)
+async def put_signals(body: SignalConfig, auth: Auth) -> SignalConfig:
+    user_scope(auth, write=True)
+    from litellm.proxy.proxy_server import llm_router
+
+    validate_signal_model(body, llm_router)
+    await signals_repository().save_config(body)
+    return body
+
+
+@router.post("/traces/signals", response_model=tuple[TraceSignals, ...])
+async def trace_signal_statuses(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceSignals, ...]:
+    user_scope(auth)
+    repo: Final = signals_repository()
+    config: Final = await repo.get_config()
+    existing: Final = await repo.traces(body.traces)
+    rows: Final = MappingProxyType({(row.trace_id, row.trace_ref): row for row in existing})
+    return tuple(
+        trace_signals(
+            trace,
+            rows.get((trace.trace_id, trace.trace_ref)),
+            config,
+        )
+        for trace in body.traces
+    )
+
+
 @router.post("/traces/findings", response_model=tuple[TraceFindingCount, ...])
 async def trace_findings(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceFindingCount, ...]:
     user_scope(auth)
@@ -501,13 +576,29 @@ async def claim(worker: WorkerAuth, protocol_version: int = 1, worker_release: s
     if worker.analysis_key_id is None:
         raise HTTPException(409, "Assign an analysis key to this worker in Lens setup")
     now: Final = datetime.now(timezone.utc)
-    await repository().heartbeat(worker.id, now.isoformat())
-    for candidate in await repository().lenses():
-        if not can_access(worker.scope, candidate.scope):
-            continue
-        if claimed := await claim_candidate(candidate, worker, now):
-            return claimed
-    return None
+    lens_repository: Final = repository()
+    await lens_repository.heartbeat(worker.id, now.isoformat())
+    return await claim_due(worker, now, lens_repository)
+
+
+async def claim_due(
+    worker: Worker,
+    now: datetime,
+    lens_repository: _ClaimRepository,
+    supports_model: Callable[[Worker, LensSettings], Awaitable[bool]] = worker_supports_model,
+) -> Claim | None:
+    after: DueLens | None = None  # rebind-ok: keyset cursor advances one page at a time
+    while True:
+        page = await lens_repository.due(worker.scope, now, CLAIM_CANDIDATES, after)
+        for candidate in page:
+            if not can_access(worker.scope, candidate.lens.scope):
+                continue
+            if claimed := await claim_candidate(candidate.lens, worker, now, lens_repository, supports_model):
+                return claimed
+            await lens_repository.sync_due(candidate.lens)
+        if len(page) < CLAIM_CANDIDATES:
+            return None
+        after = page[-1]
 
 
 @router.post("/worker/{lens_id}/{job_id}/progress", response_model=bool)
@@ -540,24 +631,37 @@ async def sample(lens_id: str, job_id: str, worker: WorkerAuth, storage: Storage
     lens, job = await assigned(lens_id, job_id, worker)
     if job.sample is not None:
         return job.sample
-    pages: list[Sample] = []  # mutable-ok: freeze selection after stable cursor traversal
+
+    async def read_page(cursor: str, sizes: tuple[int, ...]) -> tuple[Sample, tuple[int, ...]]:
+        page_size: Final = sizes[0]
+        try:
+            page: Final = await source_reader(storage).sample(
+                lens.scope,
+                job.settings,
+                int(job.start.timestamp() * 1000),
+                int(job.end.timestamp() * 1000),
+                page_size=page_size,
+                cursor=cursor,
+            )
+        except RuntimeError as error:
+            if type(error) is not RuntimeError or str(error) != SAMPLE_RESPONSE_TOO_LARGE or len(sizes) == 1:
+                raise
+            return await read_page(cursor, sizes[1:])
+        return page, sizes
+
+    pages: list[tuple[Sample, tuple[int, ...]]] = []  # mutable-ok: freeze selection after stable cursor traversal
     cursor = ""  # rebind-ok: advance by immutable identity, never by shifting row positions
     while True:
-        page = await source_reader(storage).sample(
-            lens.scope,
-            job.settings,
-            int(job.start.timestamp() * 1000),
-            int(job.end.timestamp() * 1000),
-            cursor=cursor,
-        )
-        pages.append(page)
-        if not page.next_cursor or sum(len(p.executions) for p in pages) >= pages[0].selected:
+        sizes: Final = pages[-1][1] if pages else SAMPLE_PAGE_SIZES
+        page, usable_sizes = await read_page(cursor, sizes)
+        pages.append((page, usable_sizes))
+        if not page.next_cursor or sum(len(p.executions) for p, _ in pages) >= pages[0][0].selected:
             break
         cursor = page.next_cursor
     executions: Final = tuple(
-        execution for p in pages for execution in p.executions
+        execution for p, _ in pages for execution in p.executions
     )  # comprehension-ok: flatten query pages
-    selected: Final = Sample(executions=executions, eligible=pages[0].eligible, selected=len(executions))
+    selected: Final = Sample(executions=executions, eligible=pages[0][0].eligible, selected=len(executions))
 
     def freeze(e: Lens) -> Lens:
         active: Final = current_job(e)
@@ -752,9 +856,15 @@ async def heartbeat(lens_id: str, job_id: str, worker: WorkerAuth) -> bool:
     return await progress(lens_id, job_id, Progress(), worker)
 
 
-async def claim_candidate(candidate: Lens, worker: Worker, now: datetime) -> Claim | None:
+async def claim_candidate(
+    candidate: Lens,
+    worker: Worker,
+    now: datetime,
+    lens_repository: _ClaimRepository,
+    supports_model: Callable[[Worker, LensSettings], Awaitable[bool]] = worker_supports_model,
+) -> Claim | None:
     active: Final = current_job(candidate)
-    if not await worker_supports_model(worker, active.settings if active else candidate.settings):
+    if not await supports_model(worker, active.settings if active else candidate.settings):
         return None
     job_id: Final = str(uuid4())
 
@@ -765,7 +875,7 @@ async def claim_candidate(candidate: Lens, worker: Worker, now: datetime) -> Cla
             return e
         return claim_job(scheduled, worker, now)
 
-    updated: Final = await repository().update(candidate.id, schedule, changed_only=True)
+    updated: Final = await lens_repository.update(candidate.id, schedule, attempts=1, changed_only=True)
     if updated is None:
         return None
     job: Final = current_job(updated)

litellm/proxy/lens/repository.py

@@ -3,6 +3,7 @@ import json
 import random
 from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Protocol
@@ -24,7 +25,7 @@ from litellm.proxy.lens.models import (
     Worker,
 )
 from litellm.proxy.lens.reviews import criteria_key
-from litellm.proxy.lens.state import apply_progress, current_job, replace_job
+from litellm.proxy.lens.state import apply_progress, current_job, due_at, replace_job
 from litellm.types.llms.base import LiteLLMBaseModel
 
 if TYPE_CHECKING:
@@ -39,6 +40,18 @@ class Database(Protocol):
 
 class Row(LiteLLMBaseModel):
     data: JsonValue
+    due_at: datetime | None = None
+
+
+class DueRow(LiteLLMBaseModel):
+    data: JsonValue
+    due_at: datetime
+
+
+@dataclass(frozen=True, slots=True)
+class DueLens:
+    lens: Lens
+    due_at: datetime
 
 
 class FindingRun(LiteLLMBaseModel):
@@ -47,6 +60,26 @@ class FindingRun(LiteLLMBaseModel):
 
 
 _ROWS: Final = TypeAdapter(tuple[Row, ...])
+_DUE_ROWS: Final = TypeAdapter(tuple[DueRow, ...])
+_DUE_QUERY: Final[LiteralString] = """SELECT data, due_at FROM "LiteLLM_Lens"
+WHERE due_at IS NOT NULL AND due_at <= ($4::timestamptz AT TIME ZONE 'UTC')
+AND ($1::boolean OR (
+    COALESCE((data->'scope'->>'all_teams')::boolean, false) IS NOT TRUE
+    AND COALESCE(data->'scope'->>'team_id', '')=$2
+    AND ($2 <> '' OR COALESCE(data->'scope'->>'api_key_hash', '')=$3)
+))
+ORDER BY due_at, id
+LIMIT $5"""
+_DUE_AFTER_QUERY: Final[LiteralString] = """SELECT data, due_at FROM "LiteLLM_Lens"
+WHERE due_at IS NOT NULL AND due_at <= ($4::timestamptz AT TIME ZONE 'UTC')
+AND (due_at, id) > ($6::timestamp, $7)
+AND ($1::boolean OR (
+    COALESCE((data->'scope'->>'all_teams')::boolean, false) IS NOT TRUE
+    AND COALESCE(data->'scope'->>'team_id', '')=$2
+    AND ($2 <> '' OR COALESCE(data->'scope'->>'api_key_hash', '')=$3)
+))
+ORDER BY due_at, id
+LIMIT $5"""
 UPDATE_ATTEMPTS: Final = 40
 UPDATE_BACKOFF_SECONDS: Final = 0.02
 
@@ -146,6 +179,30 @@ class LensRepository:
         rows: Final = _ROWS.validate_python(await self.db.query_raw('SELECT data FROM "LiteLLM_Lens" ORDER BY id'))
         return tuple(Lens.model_validate(row.data) for row in rows)
 
+    async def due(self, scope: Scope, now: datetime, limit: int, after: DueLens | None = None) -> tuple[DueLens, ...]:
+        query: Final[LiteralString] = _DUE_QUERY if after is None else _DUE_AFTER_QUERY
+        parameters: Final[tuple[object, ...]] = (
+            (
+                scope.all_teams,
+                scope.team_id,
+                scope.api_key_hash,
+                now.isoformat(),
+                limit,
+            )
+            if after is None
+            else (
+                scope.all_teams,
+                scope.team_id,
+                scope.api_key_hash,
+                now.isoformat(),
+                limit,
+                after.due_at,
+                after.lens.id,
+            )
+        )
+        rows: Final = _DUE_ROWS.validate_python(await self.db.query_raw(query, *parameters), from_attributes=True)
+        return tuple(DueLens(lens=Lens.model_validate(row.data), due_at=row.due_at) for row in rows)
+
     async def get(self, lens_id: str) -> Lens | None:
         rows: Final = _ROWS.validate_python(
             await self.db.query_raw(
@@ -157,12 +214,25 @@ class LensRepository:
 
     async def create(self, lens: Lens) -> Lens:
         await self.db.execute_raw(
-            'INSERT INTO "LiteLLM_Lens" (id, version, data) VALUES ($1,0,$2::jsonb)',
+            """INSERT INTO "LiteLLM_Lens" (id, version, data, due_at)
+            VALUES ($1,0,$2::jsonb,($3::timestamptz AT TIME ZONE 'UTC'))""",
             lens.id,
             lens.model_dump_json(),
+            scheduled_at.isoformat() if (scheduled_at := due_at(lens)) else None,
         )
         return lens
 
+    async def sync_due(self, lens: Lens) -> None:
+        await self.db.execute_raw(
+            """UPDATE "LiteLLM_Lens"
+            SET due_at=($3::timestamptz AT TIME ZONE 'UTC')
+            WHERE id=$1 AND version=$2
+              AND due_at IS DISTINCT FROM ($3::timestamptz AT TIME ZONE 'UTC')""",
+            lens.id,
+            lens.version,
+            scheduled_at.isoformat() if (scheduled_at := due_at(lens)) else None,
+        )
+
     async def update(
         self,
         lens_id: str,
@@ -193,7 +263,8 @@ class LensRepository:
                 """WITH previous AS MATERIALIZED (
                 SELECT data FROM "LiteLLM_Lens" WHERE id=$2 AND version=$3 FOR UPDATE
             ), updated AS (
-                UPDATE "LiteLLM_Lens" SET data=$1::jsonb, version=version+1
+                UPDATE "LiteLLM_Lens" SET data=$1::jsonb, version=version+1,
+                    due_at=($4::timestamptz AT TIME ZONE 'UTC')
                 WHERE id=$2 AND version=$3 AND EXISTS (SELECT 1 FROM previous) RETURNING id
             )
             , archived AS (INSERT INTO "LiteLLM_LensRun" (id, lens_id, created_at, data)
@@ -207,6 +278,7 @@ class LensRepository:
                 updated.model_dump_json(),
                 lens_id,
                 previous.version,
+                scheduled_at.isoformat() if (scheduled_at := due_at(updated)) else None,
             )
         )
         return bool(rows and rows[0].data == 1), updated

litellm/proxy/lens/signal_repository.py

@@ -0,0 +1,142 @@
+import json
+from datetime import datetime
+from typing import Final
+
+from pydantic import TypeAdapter
+
+from litellm.proxy.lens.models import Execution, TraceIdentity
+from litellm.proxy.lens.repository import Database, Row
+from litellm.proxy.lens.signals import (
+    SIGNAL_RECLASSIFY_AFTER,
+    SIGNAL_RETRY_FAILED_AFTER,
+    SignalAttempt,
+    SignalConfig,
+    StoredTraceSignal,
+)
+
+_ROWS: Final[TypeAdapter[tuple[Row, ...]]] = TypeAdapter(tuple[Row, ...])
+
+
+class SignalRepository:
+    def __init__(self, db: Database) -> None:
+        self.db: Final = db
+
+    async def get_config(self) -> SignalConfig:
+        rows: Final = _ROWS.validate_python(
+            await self.db.query_raw('SELECT data FROM "LiteLLM_LensSignalConfig" WHERE id=$1', "global")
+        )
+        return SignalConfig() if not rows else SignalConfig.model_validate(rows[0].data)
+
+    async def save_config(self, config: SignalConfig) -> None:
+        await self.db.execute_raw(
+            """INSERT INTO "LiteLLM_LensSignalConfig" (id, data)
+            VALUES ($1, $2::jsonb)
+            ON CONFLICT (id) DO UPDATE SET data=EXCLUDED.data""",
+            "global",
+            json.dumps(config.model_dump(mode="json")),
+        )
+
+    async def traces(self, identities: tuple[TraceIdentity, ...]) -> tuple[StoredTraceSignal, ...]:
+        if not identities:
+            return ()
+        payload: Final = json.dumps(
+            tuple({"trace_id": trace.trace_id, "trace_ref": trace.trace_ref} for trace in identities)
+        )
+        rows: Final = _ROWS.validate_python(
+            await self.db.query_raw(
+                """SELECT jsonb_build_object(
+                    'trace_id', trace_id,
+                    'trace_ref', trace_ref,
+                    'config_key', config_key,
+                    'span_count', span_count,
+                    'claimed_until', claimed_until,
+                    'classified_at', classified_at,
+                    'data', data
+                ) AS data
+                FROM "LiteLLM_LensTraceSignal"
+                WHERE (trace_id, trace_ref) IN (
+                    SELECT trace_id, trace_ref FROM jsonb_to_recordset($1::jsonb) AS requested(
+                        trace_id text, trace_ref text
+                    )
+                )""",
+                payload,
+            )
+        )
+        return tuple(StoredTraceSignal.model_validate(row.data) for row in rows)
+
+    async def claim(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        now: datetime,
+    ) -> bool:
+        data: Final = json.dumps({"status": "pending", "scores": {}, "model": config.model, "error": ""})
+        rows: Final = _ROWS.validate_python(
+            await self.db.query_raw(
+                """INSERT INTO "LiteLLM_LensTraceSignal" AS stored
+                    (trace_id, trace_ref, config_key, span_count, claimed_until, classified_at, data)
+                VALUES ($1, $2, $3, $4, $5::timestamp, NULL, $6::jsonb)
+                ON CONFLICT (trace_id, trace_ref) DO UPDATE SET
+                    config_key=EXCLUDED.config_key,
+                    span_count=EXCLUDED.span_count,
+                    claimed_until=EXCLUDED.claimed_until,
+                    classified_at=NULL,
+                    data=EXCLUDED.data
+                WHERE (stored.claimed_until IS NULL OR stored.claimed_until < $7::timestamp)
+                    AND (
+                        stored.config_key IS DISTINCT FROM EXCLUDED.config_key
+                        OR (
+                            stored.data->>'status'='pending'
+                            AND stored.claimed_until < $7::timestamp
+                        )
+                        OR (
+                            EXCLUDED.span_count > stored.span_count
+                            AND stored.classified_at < $8::timestamp
+                        )
+                        OR (
+                            stored.data->>'status'='failed'
+                            AND stored.classified_at < $9::timestamp
+                        )
+                    )
+                RETURNING jsonb_build_object('trace_id', trace_id) AS data""",
+                execution.trace_id,
+                execution.trace_ref,
+                config.key(),
+                execution.span_count,
+                claimed_until,
+                data,
+                now,
+                now - SIGNAL_RECLASSIFY_AFTER,
+                now - SIGNAL_RETRY_FAILED_AFTER,
+            )
+        )
+        return bool(rows)
+
+    async def store(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        classified_at: datetime,
+        attempt: SignalAttempt,
+    ) -> None:
+        payload: Final = json.dumps(
+            {
+                "status": attempt.status,
+                "scores": dict(attempt.scores),
+                "model": attempt.model,
+                "error": attempt.error,
+            }
+        )
+        await self.db.execute_raw(
+            """UPDATE "LiteLLM_LensTraceSignal"
+            SET classified_at=$1::timestamp, claimed_until=NULL, data=$2::jsonb
+            WHERE trace_id=$3 AND trace_ref=$4 AND config_key=$5 AND claimed_until=$6::timestamp""",
+            classified_at,
+            payload,
+            execution.trace_id,
+            execution.trace_ref,
+            config.key(),
+            claimed_until,
+        )

litellm/proxy/lens/signals.py

@@ -0,0 +1,586 @@
+import asyncio
+import hashlib
+import json
+from collections.abc import Callable, Mapping
+from datetime import datetime, timedelta, timezone
+from itertools import accumulate
+from types import MappingProxyType
+from typing import Annotated, Final, Literal, Protocol, TypeAlias
+
+from pydantic import ConfigDict, Field, JsonValue, ValidationError, field_validator, model_validator
+
+from litellm.integrations.clickhouse.context import lens_analysis
+from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
+from litellm.litellm_core_utils.secret_redaction import redact_internal_details
+from litellm.proxy.lens.models import ActivitySelection, Execution, Record, Scope, TraceIdentity
+from litellm.proxy.lens.sources import SourceReader, Storage
+
+SIGNAL_INTERVAL_SECONDS: Final = 60
+SIGNAL_PAGE_SIZE: Final = 100
+SIGNAL_MAX_PER_TICK: Final = 50
+SIGNAL_CONCURRENCY: Final = 8
+SIGNAL_CLAIM_LEASE: Final = timedelta(minutes=5)
+SIGNAL_RECLASSIFY_AFTER: Final = timedelta(minutes=5)
+SIGNAL_RETRY_FAILED_AFTER: Final = timedelta(minutes=30)
+SIGNAL_MAX_CONTENT_PAGES: Final = 3
+SIGNAL_PART_MAX_CHARS: Final = 2000
+SIGNAL_PART_HEAD_CHARS: Final = 800
+SIGNAL_PART_TAIL_CHARS: Final = 1200
+SIGNAL_TRANSCRIPT_MAX_CHARS: Final = 40000
+SIGNAL_TRANSCRIPT_HEAD_CHARS: Final = 15000
+SIGNAL_TRANSCRIPT_TAIL_CHARS: Final = 25000
+SIGNAL_MAX_SCAN_PAGES: Final = 10
+SIGNAL_TASK: Final = (
+    "An AI agent run recorded as a trace. Judge only what the user and the agent said and did in these steps."
+)
+
+
+class Signal(Record):
+    id: str = Field(pattern=r"^[a-z][a-z0-9_]{0,63}$")
+    name: str = Field(min_length=1, max_length=60)
+    question: str = Field(min_length=3, max_length=500)
+
+
+DEFAULT_SIGNALS: Final[tuple[Signal, ...]] = (
+    Signal(
+        id="user_frustration",
+        name="User frustration",
+        question=(
+            "Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example "
+            "complaints, irritated corrections, all caps, profanity, or giving up on the task?"
+        ),
+    ),
+    Signal(
+        id="missing_capability",
+        name="Missing capability",
+        question=(
+            "Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it "
+            "lacks a tool, permission, integration or data source, or fails because the capability does not exist?"
+        ),
+    ),
+    Signal(
+        id="repeated_request",
+        name="Repeated request",
+        question=(
+            "Does the user ask for the same thing more than once in this run, usually because the agent did not "
+            "deliver it the first time?"
+        ),
+    ),
+)
+
+
+class SignalConfig(Record):
+    model: str = ""
+    threshold: float = Field(default=0.5, ge=0.05, le=0.95, allow_inf_nan=False)
+    signals: tuple[Signal, ...] = DEFAULT_SIGNALS
+
+    @model_validator(mode="after")
+    def validate_signals(self) -> "SignalConfig":
+        if len(self.signals) > 20:
+            raise ValueError("A maximum of 20 signals is allowed")
+        if len(frozenset(signal.id for signal in self.signals)) != len(self.signals):
+            raise ValueError("Signal IDs must be unique")
+        return self
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.model) and bool(self.signals)
+
+    def key(self) -> str:
+        payload: Final = json.dumps(
+            {
+                "model": self.model,
+                "signals": tuple({"id": signal.id, "question": signal.question} for signal in self.signals),
+            },
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+        return hashlib.sha256(payload.encode()).hexdigest()
+
+
+Score: TypeAlias = Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)]
+
+
+class SignalFlag(Record):
+    signal_id: str
+    name: str
+    score: Score
+
+
+class TraceSignals(TraceIdentity):
+    status: Literal["unclassified", "pending", "classified", "failed"]
+    flags: tuple[SignalFlag, ...] = ()
+    model: str = ""
+    classified_at: datetime | None = None
+
+
+class SignalStep(Record):
+    kind: str
+    name: str
+    content: str
+
+
+class SignalData(Record):
+    model_config = ConfigDict(extra="ignore")
+
+    status: Literal["pending", "classified", "failed"] = "pending"
+    scores: Mapping[str, Score] = Field(default_factory=lambda: MappingProxyType({}))
+    model: str = ""
+    error: str = ""
+
+
+class SignalAttempt(Record):
+    status: Literal["classified", "failed"]
+    scores: Mapping[str, Score] = Field(default_factory=lambda: MappingProxyType({}))
+    model: str
+    error: str = ""
+
+
+class StoredTraceSignal(Record):
+    trace_id: str
+    trace_ref: str = ""
+    config_key: str
+    span_count: int
+    claimed_until: datetime | None = None
+    classified_at: datetime | None = None
+    data: JsonValue
+
+    @field_validator("claimed_until", "classified_at")
+    @classmethod
+    def normalize_database_timestamp(cls, value: datetime | None) -> datetime | None:
+        if value is not None and value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value
+
+
+class NoulAnswer(Record):
+    model_config = ConfigDict(extra="ignore", allow_inf_nan=False, from_attributes=True)
+
+    type: Literal["noul"]
+    noul: float = Field(ge=0, le=1, allow_inf_nan=False)
+
+
+class DecisionsOutput(Record):
+    model_config = ConfigDict(extra="ignore", from_attributes=True)
+
+    answers: Mapping[str, object]
+
+
+DecisionState: TypeAlias = Mapping[str, object]
+DecisionQuestions: TypeAlias = Mapping[str, Mapping[str, str]]
+Clock: TypeAlias = Callable[[], datetime]
+RouterReady: TypeAlias = Callable[[], bool]
+
+
+class DecisionsCall(Protocol):
+    async def __call__(
+        self,
+        *,
+        model: str,
+        state: DecisionState,
+        questions: DecisionQuestions,
+        timeout: float,
+        metadata: Mapping[str, object],
+    ) -> object: ...
+
+
+class SignalRepositoryProtocol(Protocol):
+    async def get_config(self) -> SignalConfig: ...
+    async def traces(self, identities: tuple[TraceIdentity, ...]) -> tuple[StoredTraceSignal, ...]: ...
+    async def claim(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        now: datetime,
+    ) -> bool: ...
+    async def store(
+        self,
+        execution: Execution,
+        config: SignalConfig,
+        claimed_until: datetime,
+        classified_at: datetime,
+        attempt: SignalAttempt,
+    ) -> None: ...
+
+
+def signal_identity(trace: TraceIdentity | StoredTraceSignal | Execution) -> tuple[str, str]:
+    return trace.trace_id, trace.trace_ref
+
+
+def candidate(
+    trace: Execution,
+    existing: StoredTraceSignal | None,
+    config_key: str,
+    now: datetime,
+) -> bool:
+    if existing is None:
+        return True
+    if existing.claimed_until is not None and existing.claimed_until > now:
+        return False
+    if existing.config_key != config_key:
+        return True
+    status: Final = existing.data.get("status") if isinstance(existing.data, dict) else ""
+    if status == "pending":
+        return existing.claimed_until is not None and existing.claimed_until <= now
+    if existing.span_count > trace.span_count:
+        return False
+    if existing.span_count < trace.span_count:
+        return existing.classified_at is not None and existing.classified_at < now - SIGNAL_RECLASSIFY_AFTER
+    return (
+        status == "failed"
+        and existing.classified_at is not None
+        and existing.classified_at < now - SIGNAL_RETRY_FAILED_AFTER
+    )
+
+
+def _take_head(steps: tuple[SignalStep, ...], remaining: int) -> tuple[SignalStep, ...]:
+    if remaining <= 0:
+        return ()
+    cumulative_lengths: Final = tuple(accumulate(len(step.content) for step in steps))
+    boundary: Final = next((index for index, total in enumerate(cumulative_lengths) if total >= remaining), None)
+    if boundary is None:
+        return steps
+    preceding: Final = steps[:boundary]
+    last: Final = steps[boundary]
+    used: Final = cumulative_lengths[boundary - 1] if boundary > 0 else 0
+    last_length: Final = remaining - used
+    return (
+        *preceding,
+        last
+        if last_length == len(last.content)
+        else last.model_copy(update=MappingProxyType({"content": last.content[:last_length]})),
+    )
+
+
+def _take_tail(steps: tuple[SignalStep, ...], remaining: int) -> tuple[SignalStep, ...]:
+    if remaining <= 0:
+        return ()
+    reversed_steps: Final = tuple(reversed(steps))
+    cumulative_lengths: Final = tuple(accumulate(len(step.content) for step in reversed_steps))
+    boundary: Final = next((index for index, total in enumerate(cumulative_lengths) if total >= remaining), None)
+    if boundary is None:
+        return steps
+    preceding: Final = reversed_steps[:boundary]
+    last: Final = reversed_steps[boundary]
+    used: Final = cumulative_lengths[boundary - 1] if boundary > 0 else 0
+    last_length: Final = remaining - used
+    selected: Final = (
+        *preceding,
+        last
+        if last_length == len(last.content)
+        else last.model_copy(update=MappingProxyType({"content": last.content[-last_length:]})),
+    )
+    return tuple(reversed(selected))
+
+
+def _bounded_steps(steps: tuple[SignalStep, ...]) -> tuple[SignalStep, ...]:
+    if sum(len(step.content) for step in steps) <= SIGNAL_TRANSCRIPT_MAX_CHARS:
+        return steps
+    head: Final = _take_head(steps, SIGNAL_TRANSCRIPT_HEAD_CHARS)
+    tail: Final = _take_tail(steps, SIGNAL_TRANSCRIPT_TAIL_CHARS)
+    omitted_count: Final = len(steps) - len(head) - len(tail)
+    marker: Final = SignalStep(kind="omitted", name="", content=f"{omitted_count} steps omitted")
+    return (*head, marker, *tail)
+
+
+def _part_excerpt(content: str) -> str:
+    if len(content) <= SIGNAL_PART_MAX_CHARS:
+        return content
+    omitted: Final = len(content) - SIGNAL_PART_MAX_CHARS
+    marker: Final = f"\n[... {omitted} characters omitted ...]\n"
+    return f"{content[:SIGNAL_PART_HEAD_CHARS]}{marker}{content[-SIGNAL_PART_TAIL_CHARS:]}"
+
+
+async def _content_pages(
+    reader: SourceReader,
+    scope: Scope,
+    execution: Execution,
+    cursor: str,
+    pages_left: int,
+) -> tuple[SignalStep, ...]:
+    if pages_left == 0:
+        return ()
+    content: Final = await reader.content(scope, execution, cursor)
+    current: Final = tuple(
+        SignalStep(kind=part.kind, name=part.name, content=_part_excerpt(part.content)) for part in content.parts
+    )
+    rest: Final = (
+        await _content_pages(reader, scope, execution, content.next_cursor, pages_left - 1)
+        if content.next_cursor is not None
+        else ()
+    )
+    return (*current, *rest)
+
+
+async def signal_state(reader: SourceReader, scope: Scope, execution: Execution) -> DecisionState:
+    steps: Final = _bounded_steps(await _content_pages(reader, scope, execution, "", SIGNAL_MAX_CONTENT_PAGES))
+    return {
+        "task": SIGNAL_TASK,
+        "steps": tuple(step.model_dump(mode="json") for step in steps),
+    }
+
+
+def _noul_score(value: object) -> float | None:
+    try:
+        return NoulAnswer.model_validate(value).noul
+    except ValidationError:
+        return None
+
+
+class SignalClassifier:
+    def __init__(self, reader: SourceReader, completion: DecisionsCall, clock: Clock) -> None:
+        self.reader: Final = reader
+        self.completion: Final = completion
+        self.clock: Final = clock
+
+    async def classify(self, scope: Scope, execution: Execution, config: SignalConfig) -> SignalAttempt:
+        try:
+            state: Final = await signal_state(self.reader, scope, execution)
+            questions: Final = {
+                signal.id: {"type": "noul", "instructions": signal.question} for signal in config.signals
+            }
+            with lens_analysis(), inherit_message_logging_privacy(True):
+                response: Final = await self.completion(
+                    model=config.model,
+                    state=state,
+                    questions=questions,
+                    timeout=60,
+                    metadata={"tags": ["litellm-lens-signals"]},
+                )
+            output: Final = DecisionsOutput.model_validate(response)
+            scores: Final = MappingProxyType(
+                {
+                    signal.id: score
+                    for signal in config.signals
+                    if (score := _noul_score(output.answers.get(signal.id))) is not None
+                }
+            )
+            if len(scores) != len(config.signals):
+                return SignalAttempt(
+                    status="failed",
+                    scores=scores,
+                    model=config.model,
+                    error="Decisions response omitted a configured noul answer",
+                )
+            return SignalAttempt(status="classified", scores=scores, model=config.model)
+        except Exception as error:
+            detail: Final = redact_internal_details(str(error))[:300]
+            return SignalAttempt(status="failed", model=config.model, error=detail)
+
+
+def trace_signals(
+    trace: TraceIdentity,
+    existing: StoredTraceSignal | None,
+    config: SignalConfig,
+) -> TraceSignals:
+    if existing is None or existing.config_key != config.key():
+        return TraceSignals(trace_id=trace.trace_id, trace_ref=trace.trace_ref, status="unclassified")
+    data: Final = SignalData.model_validate(existing.data)
+    if data.status == "pending":
+        return TraceSignals(
+            trace_id=trace.trace_id,
+            trace_ref=trace.trace_ref,
+            status="pending",
+            model=data.model,
+        )
+    if data.status == "failed" or data.error:
+        return TraceSignals(
+            trace_id=trace.trace_id,
+            trace_ref=trace.trace_ref,
+            status="failed",
+            model=data.model,
+            classified_at=existing.classified_at,
+        )
+    flags: Final = tuple(
+        sorted(
+            (
+                SignalFlag(signal_id=signal.id, name=signal.name, score=data.scores[signal.id])
+                for signal in config.signals
+                if signal.id in data.scores and data.scores[signal.id] >= config.threshold
+            ),
+            key=lambda flag: flag.score,
+            reverse=True,
+        )
+    )
+    return TraceSignals(
+        trace_id=trace.trace_id,
+        trace_ref=trace.trace_ref,
+        status="classified",
+        flags=flags,
+        model=data.model,
+        classified_at=existing.classified_at,
+    )
+
+
+async def _process_claimed(
+    classifier: SignalClassifier,
+    repository: SignalRepositoryProtocol,
+    scope: Scope,
+    execution: Execution,
+    config: SignalConfig,
+    claimed_until: datetime,
+) -> None:
+    from litellm._logging import verbose_proxy_logger
+
+    attempt: Final = await classifier.classify(scope, execution, config)
+    try:
+        await repository.store(execution, config, claimed_until, classifier.clock(), attempt)
+    except Exception as error:
+        verbose_proxy_logger.error("Lens signal result could not be stored: %s", redact_internal_details(str(error)))
+
+
+class _SignalScan:
+    def __init__(
+        self,
+        reader: SourceReader,
+        repository: SignalRepositoryProtocol,
+        scope: Scope,
+        config: SignalConfig,
+        now: datetime,
+        cursor: str,
+        limit: int,
+    ) -> None:
+        self.reader: Final = reader
+        self.repository: Final = repository
+        self.scope: Final = scope
+        self.config: Final = config
+        self.now: Final = now
+        self.cursor: str = cursor
+        self.limit: Final = limit
+        self.executions: tuple[Execution, ...] = ()
+        self.finished: bool = False
+
+    async def _read_page(self, start: int, end: int) -> tuple[tuple[Execution, ...], str | None]:
+        page_cursor: Final = self.cursor
+        sample: Final = await self.reader.sample(
+            self.scope,
+            ActivitySelection(source="traces"),
+            start,
+            end,
+            page_size=SIGNAL_PAGE_SIZE,
+            cursor=page_cursor,
+        )
+        identities: Final = tuple(
+            TraceIdentity(trace_id=trace.trace_id, trace_ref=trace.trace_ref) for trace in sample.executions
+        )
+        existing_rows: Final = await self.repository.traces(identities)
+        existing: Final = MappingProxyType({signal_identity(row): row for row in existing_rows})
+        remaining: Final = self.limit - len(self.executions)
+        all_eligible: Final = tuple(
+            execution
+            for execution in sample.executions
+            if candidate(execution, existing.get(signal_identity(execution)), self.config.key(), self.now)
+        )
+        eligible: Final = all_eligible[:remaining]
+        next_cursor: Final = page_cursor if len(all_eligible) > remaining else sample.next_cursor
+        return eligible, next_cursor
+
+    async def run(self) -> tuple[tuple[Execution, ...], str]:
+        start: Final = int((self.now - timedelta(hours=24)).timestamp() * 1000)
+        end: Final = int((self.now - timedelta(minutes=2)).timestamp() * 1000)
+        for _ in range(SIGNAL_MAX_SCAN_PAGES):
+            if self.finished or len(self.executions) >= self.limit:
+                break
+            eligible, next_cursor = await self._read_page(start, end)
+            self.executions = (*self.executions, *eligible)
+            if next_cursor is None:
+                self.cursor = ""
+                self.finished = True
+            else:
+                self.cursor = next_cursor
+        return self.executions, self.cursor
+
+
+async def _scan_pages(
+    reader: SourceReader,
+    repository: SignalRepositoryProtocol,
+    scope: Scope,
+    config: SignalConfig,
+    now: datetime,
+    cursor: str,
+    remaining: int,
+) -> tuple[tuple[Execution, ...], str]:
+    if remaining <= 0:
+        return (), cursor
+    scan: Final = _SignalScan(reader, repository, scope, config, now, cursor, remaining)
+    return await scan.run()
+
+
+async def run_signal_tick(
+    storage: Storage,
+    repository: SignalRepositoryProtocol | None,
+    completion: DecisionsCall | None,
+    clock: Clock,
+    router_ready: RouterReady = lambda: True,
+    cursor: str = "",
+) -> str:
+    if repository is None or completion is None or not router_ready():
+        return cursor
+    now: Final = clock()
+    config: Final = await repository.get_config()
+    if not config.enabled:
+        return cursor
+    reader: Final = SourceReader(storage)
+    scope: Final = Scope(all_teams=True)
+    candidates: Final = await _scan_pages(
+        reader,
+        repository,
+        scope,
+        config,
+        now,
+        cursor,
+        SIGNAL_MAX_PER_TICK,
+    )
+    executions, next_cursor = candidates
+    classifier: Final = SignalClassifier(reader, completion, clock)
+    semaphore: Final = asyncio.Semaphore(SIGNAL_CONCURRENCY)
+
+    async def process(execution: Execution) -> None:
+        from litellm._logging import verbose_proxy_logger
+
+        async with semaphore:
+            claimed_at: Final = classifier.clock()
+            claimed_until: Final = claimed_at + SIGNAL_CLAIM_LEASE
+            try:
+                claimed: Final = await repository.claim(execution, config, claimed_until, claimed_at)
+            except Exception as error:
+                verbose_proxy_logger.error("Lens signal claim failed: %s", redact_internal_details(str(error)))
+                return
+            if not claimed:
+                return
+            await _process_claimed(classifier, repository, scope, execution, config, claimed_until)
+
+    await asyncio.gather(*(process(execution) for execution in executions))
+    return next_cursor
+
+
+class _SignalLoopState:
+    def __init__(self) -> None:
+        self.cursor: str = ""
+
+
+async def run_signal_loop(
+    storage: Storage,
+    repository: SignalRepositoryProtocol | None,
+    completion: DecisionsCall | None,
+    clock: Clock = lambda: datetime.now(timezone.utc),
+    router_ready: RouterReady = lambda: True,
+) -> None:
+    from litellm._logging import verbose_proxy_logger
+
+    state: Final = _SignalLoopState()
+    while True:
+        try:
+            state.cursor = await run_signal_tick(
+                storage,
+                repository,
+                completion,
+                clock,
+                router_ready,
+                cursor=state.cursor,
+            )
+        except Exception as error:
+            verbose_proxy_logger.error("Lens signal tick failed: %s", redact_internal_details(str(error)))
+        await asyncio.sleep(SIGNAL_INTERVAL_SECONDS)

litellm/proxy/lens/sources.py

@@ -142,6 +142,7 @@ class SourceReader:
             id=execution.trace_id,
             trace_ref=execution.trace_ref,
             record_team=execution.team_id,
+            start_time=execution.start_time,
             cursor=cursor,
             offset=offset + 1,
         )
@@ -175,6 +176,7 @@ class SourceReader:
             id=execution.trace_id,
             trace_ref=execution.trace_ref,
             record_team=execution.team_id,
+            start_time=execution.start_time,
             span=evidence.span_id,
             quote=evidence.quote,
         )

litellm/proxy/lens/state.py

@@ -37,6 +37,15 @@ def current_job(lens: Lens) -> Job | None:
     return next((job for job in lens.jobs if job.status in ("queued", "running")), None)
 
 
+def due_at(lens: Lens) -> datetime | None:
+    job: Final = current_job(lens)
+    if job is None:
+        return lens.next_run_at if lens.settings.enabled else None
+    if job.status == "queued":
+        return job.created_at
+    return job.lease_until or job.created_at
+
+
 def replace_job(lens: Lens, job: Job) -> Lens:
     return lens.model_copy(
         update=MappingProxyType({"jobs": tuple(job if old.id == job.id else old for old in lens.jobs)})

litellm/proxy/proxy_server.py

@@ -581,6 +581,14 @@ from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_sp
 from litellm.proxy.image_endpoints.endpoints import router as image_router
 from litellm.proxy.lens.dataset_endpoints import router as lens_dataset_router
 from litellm.proxy.lens.endpoints import router as lens_router
+from litellm.proxy.lens.repository import WriterDatabase
+from litellm.proxy.lens.signal_repository import SignalRepository
+from litellm.proxy.lens.signals import (
+    DecisionQuestions,
+    DecisionsCall,
+    DecisionState,
+    run_signal_loop,
+)
 from litellm.proxy.list_api.common import (
     ManagementProblem,
     problem_response,
@@ -1275,6 +1283,27 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries:
     return client.writer_db
 
 
+async def _call_current_lens_signal_router(
+    *,
+    model: str,
+    state: DecisionState,
+    questions: DecisionQuestions,
+    timeout: float,
+    metadata: Mapping[str, object],
+) -> object:
+    current_router: Final = llm_router
+    if current_router is None:
+        raise RuntimeError("The proxy router is not initialized")
+    decisions: Final[DecisionsCall] = cast(DecisionsCall, current_router.adecisions)
+    return await decisions(
+        model=model,
+        state=state,
+        questions=questions,
+        timeout=timeout,
+        metadata=metadata,
+    )
+
+
 @asynccontextmanager
 async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]:
     global \
@@ -1645,12 +1674,30 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
         state: Final[ProxyLifespanState] = {"tracing_receiver": receiver}
         from litellm.proxy.admin_mcp import admin_mcp_lifespan
 
+        signal_completion: Final[DecisionsCall] = _call_current_lens_signal_router
+        signal_task: Final = (
+            asyncio.create_task(
+                run_signal_loop(
+                    receiver.storage,
+                    SignalRepository(WriterDatabase(writer_wrapper(prisma_client.db))),
+                    signal_completion,
+                    router_ready=lambda: llm_router is not None,
+                )
+            )
+            if receiver is not None and prisma_client is not None
+            else None
+        )
+
         try:
             async with AsyncExitStack() as admin_mcp_stack:
                 try:
                     await admin_mcp_stack.enter_async_context(admin_mcp_lifespan(app))
                     yield state
                 finally:
+                    if signal_task is not None:
+                        signal_task.cancel()
+                        await asyncio.gather(signal_task, return_exceptions=True)
+
                     if model_info_scheduler is not None and model_info_scheduler.running:
                         model_info_scheduler.remove_job("refresh_model_info")
                         if model_info_scheduler is not scheduler:
@@ -13951,7 +13998,7 @@ async def run_thread(
 # )
 # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)):
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
+from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.model_repository import ModelRepository
 from litellm.repositories.table_repositories import (

litellm/proxy/schema.prisma

@@ -1939,6 +1939,9 @@ model LiteLLM_Lens {
   id      String @id
   version Int @default(0)
   data    Json
+  due_at  DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone"))
+
+  @@index([due_at], map: "LiteLLM_Lens_due_at_idx")
 }
 
 model LiteLLM_LensRun {
@@ -1974,3 +1977,20 @@ model LiteLLM_LensDataset {
 
   @@id([id, revision])
 }
+
+model LiteLLM_LensSignalConfig {
+  id   String @id
+  data Json
+}
+
+model LiteLLM_LensTraceSignal {
+  trace_id      String
+  trace_ref     String @default("")
+  config_key    String
+  span_count    Int
+  claimed_until DateTime?
+  classified_at DateTime?
+  data          Json
+
+  @@id([trace_id, trace_ref])
+}

litellm/proxy/spend_tracking/baseline_accounting.py

@@ -71,12 +71,12 @@ def _complete_usage(usage: Usage | None) -> bool:
     if usage is None or usage.prompt_tokens < 0 or usage.completion_tokens < 0:
         return False
     details: Final = usage.prompt_tokens_details
-    if details is None:
+    if details is None or not hasattr(details, "cache_creation_tokens"):
         return False
     values: Final = (details.text_tokens, details.cached_tokens, details.cache_creation_tokens)
     if any(value is None or value < 0 for value in values):
         return False
-    split: Final = details.cache_creation_token_details
+    split: Final = details.cache_creation_token_details if hasattr(details, "cache_creation_token_details") else None
     writes: Final = details.cache_creation_tokens or 0
     return (
         usage.total_tokens == usage.prompt_tokens + usage.completion_tokens
@@ -133,10 +133,13 @@ def _matches(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started:
 
 
 def _ambiguous(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: float) -> bool:
-    return entry.available_at <= started < entry.expires_at and any(
-        entry.content_fingerprint in marker.lookback_content_fingerprints
-        and (entry.uncertain or entry.ttl_seconds != marker.ttl_seconds)
-        for marker in markers
+    matching: Final = tuple(
+        marker for marker in markers if entry.content_fingerprint in marker.lookback_content_fingerprints
+    )
+    return (
+        entry.available_at <= started < entry.expires_at
+        and bool(matching)
+        and (entry.uncertain or all(entry.ttl_seconds != marker.ttl_seconds for marker in matching))
     )
 
 
@@ -261,7 +264,7 @@ def _writes(history: BaselineHistory, observation: BaselineObservation) -> tuple
                 observation.started_at + hit.ttl_seconds,
             ),
         )
-        if hit is not None and all(marker.fingerprint != hit.fingerprint for marker in markers)
+        if hit is not None
         else ()
     )
     return (
@@ -277,6 +280,7 @@ def _writes(history: BaselineHistory, observation: BaselineObservation) -> tuple
                 uncertain=bool(ambiguous),
             )
             for marker in markers
+            if hit is None or marker.prefix_tokens > hit.tokens
         ),
     )
 

litellm/router.py

@@ -14582,17 +14582,21 @@ class Router:
         to the deployment that actually served the request. Every attempt therefore
         writes or clears, never just writes.
         """
+        from litellm.router_utils.baseline_request import capture_baseline_parameters
         from litellm.types.router import BaselineRouteStamp
 
         phase_attributes(routing_decision_attributes(routing_decision))
         baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None
         baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None
         router_name: Final = routing_decision.get("router_model_name") if routing_decision else None
+        caller_parameters: Final = (
+            capture_baseline_parameters(request_kwargs) if router_name and baseline_model else None
+        )
         Router._stamp_or_clear_metadata_key(
             request_kwargs=request_kwargs,
             key="_autorouter_baseline_route",
             value=(
-                BaselineRouteStamp(router_name, baseline_model, baseline_id)
+                BaselineRouteStamp(router_name, baseline_model, baseline_id, caller_parameters)
                 if router_name and baseline_model and baseline_id
                 else None
             ),

litellm/router_strategy/complexity_router/context_compaction.py

@@ -164,6 +164,11 @@ def compaction_pending(kwargs: Mapping[str, object] | None) -> bool:
     return isinstance(state, CompactionState) and state.config is not None and not _client_managed(kwargs or _EMPTY)
 
 
+def compaction_applied(kwargs: Mapping[str, object]) -> bool:
+    state: Final = kwargs.get(_STATE_KEY)
+    return isinstance(state, CompactionState) and state.summary is not None
+
+
 def _reject(model: str, reason: str) -> NoReturn:
     from litellm.exceptions import BadRequestError
 

litellm/router_utils/baseline_request.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from itertools import accumulate
+from types import MappingProxyType
+from typing import Final, cast
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm.llms.anthropic.pass_through.messages.utils import anthropic_messages_optional_param_keys
+
+CACHE_SETTINGS: Final = (
+    "system",
+    "instructions",
+    "tools",
+    "tool_choice",
+    "parallel_tool_calls",
+    "response_format",
+    "text",
+    "reasoning",
+    "reasoning_effort",
+    "thinking",
+    "verbosity",
+    "output_config",
+    "output_format",
+    "speed",
+    "prompt_cache_key",
+    "cache_key",
+    "cached_content",
+    "previous_response_id",
+    "conversation",
+    "context_management",
+    "compaction",
+)
+_GENERIC_PARAMETERS: Final = (
+    *CACHE_SETTINGS,
+    "prompt_cache_options",
+    "prompt_cache_retention",
+    "cache_control",
+    "max_tokens",
+    "max_completion_tokens",
+    "max_output_tokens",
+    "temperature",
+    "top_p",
+    "top_k",
+    "stop_sequences",
+    "enable_prompt_caching",
+    "cache_control_injection_points",
+    "drop_params",
+    "additional_drop_params",
+)
+NATIVE_ONLY_PARAMETERS: Final = tuple(
+    key
+    for key in sorted(anthropic_messages_optional_param_keys())
+    if key not in (*_GENERIC_PARAMETERS, "metadata", "stream")
+)
+BASELINE_PARAMETERS: Final = (*_GENERIC_PARAMETERS, *NATIVE_ONLY_PARAMETERS)
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_MAX_BYTES: Final = 4 * 1024 * 1024
+_MAX_NODES: Final = 32768
+_MAX_DEPTH: Final = 32
+
+
+def _json_cost(value: object, depth: int = 0) -> Iterator[int]:
+    if depth > _MAX_DEPTH:
+        yield _MAX_BYTES + 1
+    elif isinstance(value, str):
+        yield (6 if value.isascii() else 12) * len(value) + 2
+    elif isinstance(value, dict):
+        yield 2
+        for key, item in cast(dict[object, object], value).items():
+            yield from _json_cost(key, depth + 1)
+            yield from _json_cost(item, depth + 1)
+            yield 2
+    elif isinstance(value, (list, tuple)):
+        yield 2
+        for item in cast(list[object] | tuple[object, ...], value):
+            yield from _json_cost(item, depth + 1)
+            yield 1
+    elif isinstance(value, int) and value.bit_length() > 64:
+        yield _MAX_BYTES + 1
+    elif value is None or isinstance(value, (bool, int, float)):
+        yield 32
+    else:
+        yield _MAX_BYTES + 1
+
+
+def within_baseline_budget(value: object) -> bool:
+    return all(
+        size <= _MAX_BYTES and nodes <= _MAX_NODES for nodes, size in enumerate(accumulate(_json_cost(value)), 1)
+    )
+
+
+def _parameters(value: object, *, envelope: bool = False) -> dict[str, object]:
+    if not isinstance(value, Mapping):
+        return {}
+    mapping: Final = cast(Mapping[str, object], value)
+    keys: Final = (*BASELINE_PARAMETERS, "messages") if envelope else BASELINE_PARAMETERS
+    return {key: mapping[key] for key in keys if key in mapping}
+
+
+def capture_baseline_parameters(
+    kwargs: Mapping[str, object], *, include_extra_body: bool = True
+) -> Mapping[str, JsonValue] | None:
+    extra: Final = (
+        {"extra_body": _parameters(kwargs.get("extra_body"), envelope=True)}
+        if include_extra_body and "extra_body" in kwargs
+        else {}
+    )
+    parameters: Final = {**_parameters(kwargs), **extra}
+    if not within_baseline_budget(parameters):
+        return None
+    try:
+        return MappingProxyType(_JSON_OBJECT.validate_python(parameters))
+    except ValidationError:
+        return None
+
+
+def baseline_request(
+    kwargs: Mapping[str, object],
+    caller: Mapping[str, JsonValue],
+    deployment: Mapping[str, object],
+    *,
+    include_extra_body: bool = True,
+) -> Mapping[str, object] | None:
+    snapshot: Final = capture_baseline_parameters(deployment)
+    if snapshot is None:
+        return None
+    configured: Final = {
+        **_parameters(snapshot),
+        **(_parameters(snapshot.get("extra_body")) if include_extra_body else {}),
+    }
+    requested: Final = {**_parameters(caller), **(_parameters(caller.get("extra_body")) if include_extra_body else {})}
+    configured_tools: Final = configured.get("tools") or []
+    caller_tools: Final = requested.get("tools") or []
+    merged_tools: Final = (
+        {"tools": [*configured_tools, *caller_tools]}
+        if (configured_tools or caller_tools) and isinstance(configured_tools, list) and isinstance(caller_tools, list)
+        else {}
+    )
+    return MappingProxyType(
+        {
+            **{key: value for key, value in kwargs.items() if key not in (*BASELINE_PARAMETERS, "extra_body")},
+            **configured,
+            **requested,
+            **merged_tools,
+            **(
+                {"extra_body": caller.get("extra_body", snapshot.get("extra_body"))}
+                if not include_extra_body and ("extra_body" in caller or "extra_body" in snapshot)
+                else {}
+            ),
+        }
+    )

litellm/rust_bridge/trace/generated/models.py

@@ -215,6 +215,7 @@ class LensContentParams(LiteLLMBaseModel):
     source: ContentSource
     id: str
     record_team: str
+    start_time: str
     trace_ref: str
     cursor: str
     offset: int = Field(..., ge=0, le=4294967295)
@@ -232,6 +233,7 @@ class LensEvidenceParams(LiteLLMBaseModel):
     source: ContentSource
     id: str
     record_team: str
+    start_time: str
     trace_ref: str
     span: str
     quote: str

litellm/types/guardrails.py

@@ -901,6 +901,8 @@ class ContentFilterConfigModel(LiteLLMBaseModel):
 
 MCP_SECURITY_ON_VIOLATION: Final = frozenset({"block", "alert"})
 
+LoggingOnlyScope = Literal["input", "output", "both"]
+
 
 class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch update guardrails
     api_key: str | None = Field(default=None, description="API key for the guardrail service")
@@ -1142,6 +1144,14 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
         ),
     )
 
+    logging_only_scope: LoggingOnlyScope | None = Field(
+        default=None,
+        description=(
+            "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' "
+            "(default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking."
+        ),
+    )
+
     @field_validator(
         "mode",
         "default_action",
@@ -1310,6 +1320,7 @@ class GuardrailUIAddGuardrailSettings(LiteLLMBaseModel):
     supported_actions: list[str]
     supported_modes: list[str]
     supported_modes_by_provider: dict[str, list[str]]
+    providers_without_directional_logging_only_scope: tuple[str, ...]
     pii_entity_categories: list[PiiEntityCategoryMap]
     content_filter_settings: dict[str, object] | None = None
 

litellm/types/integrations/prometheus.py

@@ -416,6 +416,7 @@ def _resolve_deployment_and_latency_caller_identity_labels(
 
 class PrometheusMetricLabels:
     litellm_llm_api_latency_metric = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.API_KEY_HASH.value,
         UserAPIKeyLabelNames.API_KEY_ALIAS.value,
@@ -430,6 +431,7 @@ class PrometheusMetricLabels:
     ]
 
     litellm_llm_api_time_to_first_token_metric = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.API_KEY_HASH.value,
         UserAPIKeyLabelNames.API_KEY_ALIAS.value,
@@ -444,6 +446,7 @@ class PrometheusMetricLabels:
     ]
 
     litellm_request_total_latency_metric = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.END_USER.value,
         UserAPIKeyLabelNames.API_KEY_HASH.value,
         UserAPIKeyLabelNames.API_KEY_ALIAS.value,
@@ -516,6 +519,7 @@ class PrometheusMetricLabels:
     ]
 
     litellm_deployment_latency_per_output_token = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.MODEL_ID.value,
         UserAPIKeyLabelNames.API_BASE.value,

litellm/types/router.py

@@ -5,7 +5,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
 import datetime
 import enum
 from collections.abc import Container, Mapping, Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import (
     TYPE_CHECKING,
     Annotated,
@@ -21,7 +21,7 @@ from typing import (
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import httpx
-from pydantic import ConfigDict, Field, field_validator, model_validator
+from pydantic import ConfigDict, Field, JsonValue, field_validator, model_validator
 from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
 
 from litellm._logging import verbose_logger
@@ -1223,6 +1223,7 @@ class BaselineRouteStamp:
     router_name: str
     baseline_model: str
     baseline_deployment_id: str
+    request_parameters: Mapping[str, JsonValue] | None = field(default=None, repr=False)
 
 
 @dataclass(frozen=True, slots=True)

model_prices_and_context_window.json

@@ -15229,8 +15229,8 @@
         "cache_creation_input_token_cost": 2.5e-06,
         "cache_creation_input_token_cost_above_1hr": 4e-06,
         "cache_creation_input_token_cost_batches": 1.25e-06,
-        "cache_read_input_token_cost": 2e-07,
-        "cache_read_input_token_cost_batches": 1e-07,
+        "cache_read_input_token_cost": 1e-07,
+        "cache_read_input_token_cost_batches": 5e-08,
         "input_cost_per_token": 2e-06,
         "input_cost_per_token_batches": 1e-06,
         "litellm_provider": "anthropic",
@@ -15267,7 +15267,7 @@
         "supports_forced_tool_use": false,
         "thinking_always_on": true,
         "prompt_cache_min_tokens": 512,
-        "source": "https://platform.claude.com/docs/en/models/sonnet-5-5/overview",
+        "source": "https://platform.claude.com/docs/en/about-claude/pricing",
         "supports_web_search": true
     },
     "claude-sonnet-4-6": {
@@ -80768,5 +80768,648 @@
         "supports_system_messages": true,
         "supports_tool_choice": true,
         "supports_vision": true
+    },
+"claude-haiku-5-5": {
+        "supports_anthropic_compaction": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_creation_input_token_cost_batches": 6.25e-08,
+        "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_batches": 5e-09,
+        "input_cost_per_token": 1e-07,
+        "input_cost_per_token_batches": 5e-08,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "output_cost_per_token_batches": 2.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_native_structured_output": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview",
+        "supports_web_search": true,
+        "input_cost_per_token_above_100k_tokens": 5e-07,
+        "output_cost_per_token_above_100k_tokens": 2.5e-06,
+        "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
+        "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
+        "cache_read_input_token_cost_above_100k_tokens": 5e-08
+    },
+    "bedrock_mantle/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_mantle",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
+    },
+    "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock_mantle",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
+    },
+    "anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
+    },
+    "apac.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "input_cost_per_token": 1.1e-07,
+        "output_cost_per_token": 5.5e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
+    },
+    "au.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "azure_ai/claude-haiku-5-5": {
+        "supports_mid_conversation_system": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "deprecation_date": "2027-09-29",
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_forced_tool_use": true,
+        "thinking_always_on": false,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
+    },
+    "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "eu.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "global.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "jp.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "perplexity/anthropic/claude-haiku-5-5": {
+        "litellm_provider": "perplexity",
+        "mode": "responses",
+        "supports_adaptive_thinking": true,
+        "supports_web_search": true,
+        "supports_function_calling": true,
+        "input_cost_per_token": 1e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 1e-08,
+        "source": "https://docs.perplexity.ai/docs/agent-api/models"
+    },
+    "us-gov.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "cache_creation_input_token_cost": 1.5e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.4e-07,
+        "cache_read_input_token_cost": 1.2e-08,
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_mid_conversation_system": true,
+        "supports_native_structured_output": false,
+        "supports_output_config": true,
+        "supports_parallel_tool_use_config": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "us.anthropic.claude-haiku-5-5": {
+        "bedrock_converse_supports_strict_tools": false,
+        "cache_creation_input_token_cost": 1.375e-07,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-07,
+        "cache_read_input_token_cost": 1.1e-08,
+        "input_cost_per_token": 1.1e-07,
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_mid_conversation_system": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": false,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh",
+        "supports_parallel_tool_use_config": true,
+        "supports_forced_tool_use": false,
+        "thinking_always_on": true,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
+    "vertex_ai/claude-haiku-5-5": {
+        "regional_endpoint_uplift_multiplier": 1.1,
+        "supports_mid_conversation_system": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_creation_input_token_cost_batches": 6.25e-08,
+        "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_batches": 5e-09,
+        "input_cost_per_token": 1e-07,
+        "input_cost_per_token_batches": 5e-08,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "output_cost_per_token_batches": 2.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_forced_tool_use": true,
+        "thinking_always_on": false,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
+    },
+    "vertex_ai/claude-haiku-5-5@default": {
+        "regional_endpoint_uplift_multiplier": 1.1,
+        "supports_mid_conversation_system": true,
+        "cache_creation_input_token_cost": 1.25e-07,
+        "cache_creation_input_token_cost_above_1hr": 2e-07,
+        "cache_creation_input_token_cost_batches": 6.25e-08,
+        "cache_read_input_token_cost": 1e-08,
+        "cache_read_input_token_cost_batches": 5e-09,
+        "input_cost_per_token": 1e-07,
+        "input_cost_per_token_batches": 5e-08,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "output_cost_per_token_batches": 2.5e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_forced_tool_use": true,
+        "thinking_always_on": false,
+        "prompt_cache_min_tokens": 512,
+        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     }
 }

model_prices_and_context_window.schema.json

@@ -83,6 +83,11 @@
           "minimum": 0,
           "description": "USD per token written to the provider's prompt cache."
         },
+        "cache_creation_input_token_cost_above_100k_tokens": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Rate applied once the prompt exceeds the token threshold in the field name."
+        },
         "cache_creation_input_token_cost_above_128k_tokens": {
           "type": "number",
           "minimum": 0,
@@ -93,6 +98,11 @@
           "minimum": 0,
           "description": "Rate applied once the prompt exceeds the token threshold in the field name."
         },
+        "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Rate applied once the prompt exceeds the token threshold in the field name."
+        },
         "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {
           "type": "number",
           "minimum": 0,
@@ -174,6 +184,11 @@
           "minimum": 0,
           "description": "USD per prompt token served from the provider's prompt cache."
         },
+        "cache_read_input_token_cost_above_100k_tokens": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Rate applied once the prompt exceeds the token threshold in the field name."
+        },
         "cache_read_input_token_cost_above_128k_tokens": {
           "type": "number",
           "minimum": 0,
@@ -377,6 +392,11 @@
           "minimum": 0,
           "description": "USD per prompt token."
         },
+        "input_cost_per_token_above_100k_tokens": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Rate applied once the prompt exceeds the token threshold in the field name."
+        },
         "input_cost_per_token_above_128k_tokens": {
           "type": "number",
           "minimum": 0,
@@ -756,6 +776,11 @@
           "minimum": 0,
           "description": "USD per generated token."
         },
+        "output_cost_per_token_above_100k_tokens": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Rate applied once the prompt exceeds the token threshold in the field name."
+        },
         "output_cost_per_token_above_128k_tokens": {
           "type": "number",
           "minimum": 0,

schema.prisma

@@ -1939,6 +1939,9 @@ model LiteLLM_Lens {
   id      String @id
   version Int @default(0)
   data    Json
+  due_at  DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone"))
+
+  @@index([due_at], map: "LiteLLM_Lens_due_at_idx")
 }
 
 model LiteLLM_LensRun {
@@ -1974,3 +1977,20 @@ model LiteLLM_LensDataset {
 
   @@id([id, revision])
 }
+
+model LiteLLM_LensSignalConfig {
+  id   String @id
+  data Json
+}
+
+model LiteLLM_LensTraceSignal {
+  trace_id      String
+  trace_ref     String @default("")
+  config_key    String
+  span_count    Int
+  claimed_until DateTime?
+  classified_at DateTime?
+  data          Json
+
+  @@id([trace_id, trace_ref])
+}

scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json

@@ -39,6 +39,9 @@
     "source": {
       "$ref": "#/$defs/ContentSource"
     },
+    "start_time": {
+      "type": "string"
+    },
     "team": {
       "type": "string"
     },
@@ -53,6 +56,7 @@
     "source",
     "id",
     "record_team",
+    "start_time",
     "trace_ref",
     "cursor",
     "offset"

scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json

@@ -36,6 +36,9 @@
     "span": {
       "type": "string"
     },
+    "start_time": {
+      "type": "string"
+    },
     "team": {
       "type": "string"
     },
@@ -50,6 +53,7 @@
     "source",
     "id",
     "record_team",
+    "start_time",
     "trace_ref",
     "span",
     "quote"

tests/code_coverage_tests/recursive_detector.py

@@ -2,6 +2,7 @@ import ast
 import os
 
 IGNORE_FUNCTIONS = [
+    "_json_cost",  # bounded at depth 32 and consumed under byte/node limits.
     "_format_type",
     "remove_additional_properties",
     "remove_strict_from_schema",

tests/e2e/AGENTS.md

@@ -215,7 +215,7 @@ llm.<endpoint>.<route>.<capability>.<streaming>.<assertion>
                | rerank | images_generations | audio_speech | audio_transcriptions | moderations
                | realtime
   route      : openai | azure_openai | anthropic | bedrock_converse | bedrock_invoke | vertex
-               | azure_foundry | cohere | together_ai
+               | azure_foundry | cohere | together_ai | ollama | ollama_chat
                (vocab varies per endpoint; messages is anthropic-format only)
   capability : basic | tool_use | prompt_cache_5m | vision | thinking | structured_output
                | service_tier | mid_conversation_system

tests/e2e/a2a/a2a_client.py

@@ -19,6 +19,7 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from e2e_config import settle_propagation
 from e2e_http import NoBody, Result, Success, get_external, is_ok
+from e2e_metadata import STEP_FRAMES, step
 from proxy_client import ProxyClient
 
 
@@ -87,7 +88,7 @@ class A2ABridgeParams(BaseModel):
 
     custom_llm_provider: str
     model: str
-    api_key: str | None = None
+    api_key: str | None = Field(default=None, repr=False)
 
 
 class AgentRegisterBody(BaseModel):
@@ -291,6 +292,7 @@ class A2AResponse(BaseModel):
 class A2AClient:
     proxy: ProxyClient
 
+    @step("Register the A2A agent {body.agent_name} through /v1/agents")
     def register_agent(self, body: AgentRegisterBody) -> Result[AgentResponse]:
         """Register an agent and, on success, wait until the data plane serves it.
 
@@ -337,6 +339,7 @@ class A2AClient:
                 )
             time.sleep(self.proxy.poll_interval)
 
+    @step("Read the A2A agent back from /v1/agents/{{agent_id}}")
     def get_agent(self, agent_id: str) -> Result[AgentResponse]:
         return self.proxy.transport.get(
             f"/v1/agents/{agent_id}",
@@ -345,6 +348,7 @@ class A2AClient:
             response_type=AgentResponse,
         )
 
+    @step("Delete the A2A agent")
     def delete_agent(self, agent_id: str) -> None:
         result = self.proxy.transport.delete(
             f"/v1/agents/{agent_id}",
@@ -353,8 +357,9 @@ class A2AClient:
             response_type=NoBody,
         )
         if not is_ok(result):
-            warnings.warn(f"delete_agent({agent_id!r}) failed: {result}", stacklevel=2)
+            warnings.warn(f"delete_agent({agent_id!r}) failed: {result}", stacklevel=2 + STEP_FRAMES)
 
+    @step("Read the A2A agent's card from /a2a/{{agent_id}}/.well-known/agent-card.json with the given key")
     def agent_card(self, agent_id: str, key: str) -> Result[ServedAgentCard]:
         return self.proxy.transport.get(
             f"/a2a/{agent_id}/.well-known/agent-card.json",
@@ -363,6 +368,7 @@ class A2AClient:
             response_type=ServedAgentCard,
         )
 
+    @step("Send an A2A message to /a2a/{{agent_id}} with {body.params.message.parts}")
     def send_message(self, agent_id: str, key: str, body: A2AJsonRpcRequest) -> Result[A2AResponse]:
         return self.proxy.transport.post(
             f"/a2a/{agent_id}",
@@ -376,6 +382,7 @@ def build_a2a_client(proxy: ProxyClient) -> A2AClient:
     return A2AClient(proxy=proxy)
 
 
+@step("Fetch a published A2A agent card from its /.well-known endpoint")
 def fetch_agent_card(url: str, *, timeout: float = 20.0) -> Result[UpstreamAgentCard]:
     """Fetch a live A2A agent card from its /.well-known endpoint and parse it into the
     registration model, so a test can register a real published card verbatim rather

tests/e2e/a2a/test_a2a_agent_e2e.py

@@ -10,6 +10,8 @@ protocol version, and an unsupported version is refused at registration).
 
 from __future__ import annotations
 
+from typing import Final
+
 import pytest
 
 from a2a_client import (
@@ -31,6 +33,9 @@ from a2a_client import (
 from e2e_config import unique_marker
 from e2e_http import Result, UnknownApiError, unwrap
 from lifecycle import ResourceManager
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
+
+BRIDGE_MODEL: Final = "claude-haiku-4-5"
 
 # No api_key: litellm resolves ANTHROPIC_API_KEY from the proxy's own environment
 # for this provider, which is what the agent-owner flow relies on. Pinning
@@ -42,7 +47,7 @@ from lifecycle import ResourceManager
 # omitted -> 200, "os.environ/..." -> 500 invalid x-api-key, literal key -> 200.
 BRIDGE = A2ABridgeParams(
     custom_llm_provider="anthropic",
-    model="claude-haiku-4-5",
+    model=BRIDGE_MODEL,
 )
 
 MOVEHOME_AGENT_CARD_URL = "https://movehome.org/.well-known/agent.json"
@@ -96,6 +101,12 @@ def _ask(text: str) -> A2AJsonRpcRequest:
 
 class TestA2AAgentLifecycle:
     @pytest.mark.covers("other.a2a.register.persists")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+        )
+    )
     def test_register_persists(self, client: A2AClient, resources: ResourceManager) -> None:
         agent = _register(client, resources, "0.3")
         fetched = unwrap(client.get_agent(agent.agent_id))
@@ -104,6 +115,15 @@ class TestA2AAgentLifecycle:
         assert fetched.agent_card_params.protocol_version == "0.3"
 
     @pytest.mark.covers("other.a2a.register.semver_version_accepted")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+            providers=(Provider.ANTHROPIC,),
+            models=(BRIDGE_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_semver_protocol_version_registers_and_serves(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
         agent = _register(client, resources, "0.3.0")
         assert agent.agent_card_params.protocol_version == "0.3"
@@ -116,6 +136,12 @@ class TestA2AAgentLifecycle:
         assert result.text != ""
 
     @pytest.mark.covers("other.a2a.message_send.real_world_agent_replies")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+        )
+    )
     def test_real_world_agent_replies_to_property_query(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
         upstream = unwrap(fetch_agent_card(MOVEHOME_AGENT_CARD_URL)).model_copy(update={"url": MOVEHOME_ORIGIN})
         assert upstream.protocol_version == "0.3.0"
@@ -152,6 +178,12 @@ class TestA2AAgentLifecycle:
         assert all(listing.location.un_locode == location for listing in results.listings)
 
     @pytest.mark.covers("other.a2a.discovery.proxy_fronted_card")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+        )
+    )
     def test_discovery_card_is_proxy_fronted(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
         agent = _register(client, resources, "0.3")
         card = unwrap(client.agent_card(agent.agent_id, scoped_key))
@@ -163,6 +195,15 @@ class TestA2AAgentLifecycle:
         assert card.supported_interfaces[0].url == card.url
 
     @pytest.mark.covers("other.a2a.message_send.bridge_invokes")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+            providers=(Provider.ANTHROPIC,),
+            models=(BRIDGE_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_message_send_runs_completion_bridge(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
         agent = _register(client, resources, "0.3")
         request = _ask("Reply with exactly the word PONG and nothing else")
@@ -177,6 +218,15 @@ class TestA2AAgentLifecycle:
         assert rows[0].model == f"a2a_agent/{agent.agent_card_params.name}"
 
     @pytest.mark.covers("other.a2a.version.serves_pinned_0_3")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+            providers=(Provider.ANTHROPIC,),
+            models=(BRIDGE_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_pinned_v0_3_serves_flat_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
         agent = _register(client, resources, "0.3")
         request = _ask("Say hi in one word")
@@ -188,6 +238,15 @@ class TestA2AAgentLifecycle:
         assert result.text != ""
 
     @pytest.mark.covers("other.a2a.version.serves_pinned_1_0")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+            providers=(Provider.ANTHROPIC,),
+            models=(BRIDGE_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_pinned_v1_0_serves_nested_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
         agent = _register(client, resources, "1.0")
         request = _ask("Say hi in one word")
@@ -199,6 +258,12 @@ class TestA2AAgentLifecycle:
         assert result.text != ""
 
     @pytest.mark.covers("other.a2a.register.unsupported_version_rejected")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+        )
+    )
     def test_unsupported_protocol_version_rejected(self, client: A2AClient) -> None:
         result = _register_rejection(client, "9.9")
         match result:
@@ -209,6 +274,12 @@ class TestA2AAgentLifecycle:
                 pytest.fail(f"expected 400 for unsupported protocolVersion, got {result}")
 
     @pytest.mark.covers("other.a2a.register.malformed_version_rejected")
+    @meta(
+        Subject(
+            domain=Domain.AGENTS_API,
+            route=Route.A2A,
+        )
+    )
     def test_malformed_protocol_version_rejected(self, client: A2AClient) -> None:
         result = _register_rejection(client, "0.3.garbage")
         match result:

tests/e2e/access_control/access_control_client.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass
 from pydantic import BaseModel, ValidationError
 
 from proxy_client import ProxyClient
+from e2e_metadata import step
 from e2e_http import NoBody, StreamingResponse, is_ok, unwrap
 from models import (
     ChatBody,
@@ -59,14 +60,17 @@ def error_envelope(body: str) -> ApiErrorEnvelope | None:
 class AccessControlClient:
     proxy: ProxyClient
 
+    @step("Generate a virtual key that can only call LLM API routes")
     def llm_only_key(self) -> str:
         return self.proxy.generate_key(
             KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"])
         )
 
+    @step("Delete the virtual key")
     def delete_key(self, key: str) -> None:
         self.proxy.delete_key(key)
 
+    @step('Send a /chat/completions request to {model} with the prompt "{content}"')
     def chat_status(
         self, key: str, model: str, content: str, max_completion_tokens: int | None = None
     ) -> StreamingResponse:
@@ -80,6 +84,7 @@ class AccessControlClient:
             ),
         )
 
+    @step("Create the team {team_alias} with models: {models}")
     def create_team(self, team_alias: str, models: list[str]) -> str:
         team_id = unwrap(
             self.proxy.transport.post(
@@ -92,6 +97,7 @@ class AccessControlClient:
         self._await_team(team_id)
         return team_id
 
+    @step("Set the team {team_alias}'s models to {models} through /team/update")
     def set_team_models(self, team_id: str, team_alias: str, models: list[str]) -> None:
         """Replace the team's allow-list. /model/new appends a team-scoped deployment's
         public name to it, so a test that means to grant only an access group has to
@@ -105,6 +111,7 @@ class AccessControlClient:
             )
         )
 
+    @step("Delete the team")
     def delete_team(self, team_id: str) -> None:
         _ = self.proxy.transport.post(
             "/team/delete",
@@ -113,6 +120,7 @@ class AccessControlClient:
             response_type=NoBody,
         )
 
+    @step("List the deployments in the model access group {access_group}")
     def access_group_info(self, access_group: str) -> AccessGroupInfoResponse | None:
         result = self.proxy.transport.get(
             f"/access_group/{access_group}/info",
@@ -122,6 +130,7 @@ class AccessControlClient:
         )
         return unwrap(result) if is_ok(result) else None
 
+    @step("Read the team's models from /team/info")
     def team_models(self, team_id: str) -> list[str] | None:
         result = self.proxy.transport.get(
             "/team/info",
@@ -139,6 +148,7 @@ class AccessControlClient:
             time.sleep(self.proxy.poll_interval)
         raise AssertionError(f"/team/info never resolved team {team_id!r} created by /team/new")
 
+    @step("Add a deployment named {model_name} that calls openai/gpt-4o-mini with the given key")
     def create_model_status(self, key: str, model_name: str) -> StreamingResponse:
         return self.proxy.transport.send(
             "/model/new",

tests/e2e/access_control/test_access_control_e2e.py

@@ -26,6 +26,7 @@ from e2e_http import Success, UnauthorizedError, UnknownApiError, unwrap
 from lifecycle import ResourceManager
 from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, LiteLLMParamsBody
 from proxy_client import ProxyClient
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 
 pytestmark = pytest.mark.e2e
 
@@ -36,6 +37,14 @@ EMBEDDING_MODEL = "openai-text-embedding-3-small"
 
 
 class TestAccessControl:
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            providers=(Provider.GEMINI,),
+            models=(ALLOWED_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_allowed_model_is_permitted(
         self, client: AccessControlClient, resources: ResourceManager
     ) -> None:
@@ -57,6 +66,13 @@ class TestAccessControl:
             f"200 must carry a real completion, not an error envelope: {result.body[:300]}"
         )
 
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            models=(DISALLOWED_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_disallowed_model_is_denied_403(
         self, client: AccessControlClient, resources: ResourceManager
     ) -> None:
@@ -73,6 +89,13 @@ class TestAccessControl:
         )
 
     @pytest.mark.covers("other.auth.virtual_key.route_group_allowed")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            providers=(Provider.GEMINI, Provider.OPENAI,),
+            models=(ALLOWED_MODEL, EMBEDDING_MODEL,),
+        )
+    )
     def test_llm_api_routes_group_grants_every_llm_endpoint(
         self, client: AccessControlClient, resources: ResourceManager
     ) -> None:
@@ -97,6 +120,12 @@ class TestAccessControl:
             f"the same key must still be shut out of /model/new, got {denied.status_code}: {denied.body[:300]}"
         )
 
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            route=Route.MODEL_MANAGEMENT,
+        )
+    )
     def test_llm_only_key_forbidden_from_management_route_403(
         self, client: AccessControlClient, resources: ResourceManager
     ) -> None:
@@ -111,6 +140,12 @@ class TestAccessControl:
             f"403 body must be a route-permission denial, got: {result.body[:300]}"
         )
 
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_unknown_model_returns_400(
         self, client: AccessControlClient, resources: ResourceManager
     ) -> None:
@@ -140,6 +175,14 @@ class TestVirtualKeyAuth:
         "mgmt.virtual_key.invalid_denied",
         exercised_on=[],
     )
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            providers=(Provider.ANTHROPIC,),
+            models=(VIRTUAL_KEY_BACKEND,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_valid_key_allows_and_invalid_key_denied(
         self, proxy: ProxyClient, resources: ResourceManager
     ) -> None:

tests/e2e/access_control/test_chat_auth_headers_e2e.py

@@ -11,6 +11,7 @@ import pytest
 from e2e_http import AuthHeaders, NoBody, StreamingResponse, assert_auth_denied
 from models import ChatBody, ChatMessage
 from proxy_client import ProxyClient
+from e2e_metadata import Domain, Route, Subject, meta
 
 pytestmark = pytest.mark.e2e
 
@@ -32,26 +33,56 @@ def _chat_with_headers(proxy: ProxyClient, headers: AuthHeaders | NoBody) -> Str
 
 class TestChatAuthHeaders:
     @pytest.mark.covers("other.auth.llm_chat.missing_header_denied")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            route=Route.CHAT_COMPLETIONS,
+        )
+    )
     def test_missing_authorization_header_is_denied(self, proxy: ProxyClient) -> None:
         result = _chat_with_headers(proxy, NoBody())
         assert_auth_denied(result, "missing Authorization")
 
     @pytest.mark.covers("other.auth.llm_chat.invalid_bearer_denied")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            route=Route.CHAT_COMPLETIONS,
+        )
+    )
     def test_bearer_invalid_token_is_denied(self, proxy: ProxyClient) -> None:
         result = _chat_with_headers(proxy, AuthHeaders(authorization="Bearer invalid_token"))
         assert_auth_denied(result, "Bearer invalid_token")
 
     @pytest.mark.covers("other.auth.llm_chat.no_bearer_prefix_denied")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            route=Route.CHAT_COMPLETIONS,
+        )
+    )
     def test_token_without_bearer_prefix_is_denied(self, proxy: ProxyClient) -> None:
         result = _chat_with_headers(proxy, AuthHeaders(authorization="invalid_token"))
         assert_auth_denied(result, "token without Bearer prefix")
 
     @pytest.mark.covers("other.auth.llm_chat.empty_bearer_denied")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            route=Route.CHAT_COMPLETIONS,
+        )
+    )
     def test_empty_bearer_token_is_denied(self, proxy: ProxyClient) -> None:
         result = _chat_with_headers(proxy, AuthHeaders(authorization="Bearer "))
         assert_auth_denied(result, "empty Bearer token")
 
     @pytest.mark.covers("other.auth.llm_chat.not_bearer_scheme_denied")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            route=Route.CHAT_COMPLETIONS,
+        )
+    )
     def test_not_bearer_scheme_is_denied(self, proxy: ProxyClient) -> None:
         result = _chat_with_headers(proxy, AuthHeaders(authorization="NotBearer validtoken123"))
         assert_auth_denied(result, "NotBearer scheme")

tests/e2e/access_control/test_model_access_group_e2e.py

@@ -33,6 +33,7 @@ from models import (
     ModelNewBody,
     TeamInfoResponse,
 )
+from e2e_metadata import Domain, Mode, Provider, Subject, meta
 
 pytestmark = pytest.mark.e2e
 
@@ -177,6 +178,14 @@ class TestKeyScopedToAccessGroup:
         "other.auth.model_access_group.member_allowed",
     )
     @pytest.mark.parametrize(("case", "select_model"), ALLOWED)
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            providers=(Provider.OPENAI,),
+            models=(GROUP_BACKEND, WILDCARD_BARE_MODEL, WILDCARD_PREFIXED_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_group_grants_every_deployment_in_it(
         self,
         case: str,
@@ -202,6 +211,13 @@ class TestKeyScopedToAccessGroup:
 
     @pytest.mark.covers("other.auth.model_access_group.non_member_denied")
     @pytest.mark.parametrize(("case", "select_model"), DENIED)
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            models=(GROUP_BACKEND, UNCOVERED_OPENAI_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_group_grants_nothing_outside_it(
         self,
         case: str,
@@ -228,6 +244,14 @@ class TestKeyScopedToAccessGroup:
 
 class TestTeamScopedToAccessGroup:
     @pytest.mark.covers("other.auth.model_access_group.team_wildcard_bare_name_allowed")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            providers=(Provider.OPENAI,),
+            models=(TEAM_WILDCARD_BARE_MODEL,),
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_group_grants_the_teams_own_wildcard(
         self, client: AccessControlClient, team_grant: TeamGrant
     ) -> None:
@@ -248,6 +272,12 @@ class TestTeamScopedToAccessGroup:
         )
 
     @pytest.mark.covers("other.auth.model_access_group.team_non_member_denied")
+    @meta(
+        Subject(
+            domain=Domain.PROXY_AUTH,
+            mode=Mode.NONSTREAM,
+        )
+    )
     def test_group_grants_the_team_nothing_outside_it(
         self, client: AccessControlClient, team_grant: TeamGrant
     ) -> None:

tests/e2e/batches/batch_cleanup.py

@@ -8,6 +8,7 @@ from typing import Final, Protocol
 from batch_client import BatchObject, FileDeleteResponse
 from capabilities import is_cloud_storage_id, is_managed_id
 from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApiError
+from e2e_metadata import STEP_FRAMES, step
 from pydantic import BaseModel
 
 CLEANUP_DELAYS: Final = (1.0, 2.0, 4.0)
@@ -52,6 +53,7 @@ def _require_cleanup_success[R: BaseModel](result: Result[R], operation: str) ->
             raise AssertionError(f"{operation} failed: {result.kind}")
 
 
+@step("Clean up the uploaded file")
 def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider: str | None = None) -> None:
     delete: Final[Callable[[], Result[FileDeleteResponse]]] = (
         (lambda: client.delete_file_as_admin(file_id, provider=provider))
@@ -65,7 +67,7 @@ def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider
         warnings.warn(
             f"Left file {file_id} in place: LiteLLM refused to delete it while a batch still references it",
             UserWarning,
-            stacklevel=2,
+            stacklevel=2 + STEP_FRAMES,
         )
         return
     deleted: Final = _require_cleanup_success(result, f"Delete file {file_id}")
@@ -74,6 +76,7 @@ def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider
     ), f"Delete file {file_id} did not confirm deletion"
 
 
+@step("Cancel the batch if it is still running")
 def cleanup_batch(
     client: BatchCleanupClient,
     batch_id: str,
@@ -137,7 +140,7 @@ def cleanup_batch(
             warnings.warn(
                 f"Left batch {batch_id} cancelling after {BATCH_CANCEL_TIMEOUT_SECONDS}s for the provider to finish",
                 UserWarning,
-                stacklevel=2,
+                stacklevel=2 + STEP_FRAMES,
             )
             return
         wait(BATCH_CANCEL_POLL_SECONDS)

tests/e2e/batches/batch_client.py

@@ -17,6 +17,7 @@ from typing import Final, Literal
 
 from pydantic import BaseModel, Field
 
+from e2e_metadata import step
 from proxy_client import ProxyClient
 from e2e_http import (
     FileUploadForm,
@@ -136,12 +137,15 @@ def is_result_access_denied[R: BaseModel](result: Result[R]) -> bool:
 class BatchClient:
     proxy: ProxyClient
 
+    @step("Add a batch deployment named {model_name} that calls {litellm_params.model}")
     def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str:
         return self.proxy.create_model(model_name, litellm_params, mode="batch")
 
+    @step("Delete the batch deployment")
     def delete_model(self, model_id: str) -> None:
         self.proxy.delete_model(model_id)
 
+    @step("Upload a batch input file to /v1/files")
     def upload_file(
         self,
         *,
@@ -161,6 +165,7 @@ class BatchClient:
             response_type=FileObject,
         )
 
+    @step("Retrieve the uploaded file")
     def retrieve_file(
         self, file_id: str, *, key: str, provider: str | None = None
     ) -> Result[FileObject]:
@@ -171,6 +176,7 @@ class BatchClient:
             response_type=FileObject,
         )
 
+    @step("List the files the key can see from /v1/files")
     def list_files(self, *, key: str, provider: str | None = None) -> Result[FileList]:
         return self.proxy.transport.get(
             _files_path(provider),
@@ -179,6 +185,7 @@ class BatchClient:
             response_type=FileList,
         )
 
+    @step("Create a batch of {body.endpoint} requests from the uploaded file")
     def create_batch(
         self, *, body: BatchCreateBody, key: str, provider: str | None = None
     ) -> StreamingResponse:
@@ -188,6 +195,7 @@ class BatchClient:
             json=body,
         )
 
+    @step("Retrieve the batch")
     def retrieve_batch(
         self, batch_id: str, *, key: str, provider: str | None = None
     ) -> Result[BatchObject]:
@@ -198,6 +206,7 @@ class BatchClient:
             response_type=BatchObject,
         )
 
+    @step("Cancel the batch")
     def cancel_batch(
         self, batch_id: str, *, key: str, provider: str | None = None
     ) -> Result[BatchObject]:
@@ -208,6 +217,7 @@ class BatchClient:
             response_type=BatchObject,
         )
 
+    @step("List the batches the key can see from /v1/batches")
     def list_batches(
         self,
         *,
@@ -223,6 +233,7 @@ class BatchClient:
             response_type=BatchList,
         )
 
+    @step("Delete the uploaded file")
     def delete_file(
         self, file_id: str, *, key: str, provider: str | None = None
     ) -> Result[FileDeleteResponse]:
@@ -233,6 +244,7 @@ class BatchClient:
             response_type=FileDeleteResponse,
         )
 
+    @step("Delete the uploaded file as the proxy admin")
     def delete_file_as_admin(self, file_id: str, *, provider: str | None = None) -> Result[FileDeleteResponse]:
         return self.proxy.transport.delete(
             f"{_files_path(provider)}/{file_id}",

tests/e2e/batches/capabilities.py

@@ -7,7 +7,11 @@ import os
 from dataclasses import dataclass
 from typing import Final, Literal
 
+import pytest
+
 from e2e_config import provider_edge_base, unique_marker
+from e2e_metadata import Domain, Mode, Route, Subject, meta
+from e2e_metadata import Provider as MetaProvider
 from models import LiteLLMParamsBody
 
 _BATCH_RUN = unique_marker()
@@ -18,6 +22,9 @@ def batch_model_name(base: str) -> str:
 
 
 OPENAI_BATCH_BACKEND: Final = "gpt-4o-mini"
+AZURE_BATCH_BACKEND: Final = "gpt-5.4-mini-batch"
+VERTEX_BATCH_BACKEND: Final = "gemini-2.5-flash"
+BEDROCK_BATCH_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
 
 
 def openai_batch_params() -> LiteLLMParamsBody:
@@ -65,14 +72,14 @@ class Provider:
                 return openai_batch_params()
             case "azure":
                 return LiteLLMParamsBody(
-                    model="azure/gpt-5.4-mini-batch",
+                    model=f"azure/{AZURE_BATCH_BACKEND}",
                     api_base="os.environ/AZURE_API_BASE",
                     api_key="os.environ/AZURE_API_KEY",
                     api_version="2025-04-01-preview",
                 )
             case "vertex_ai":
                 return LiteLLMParamsBody(
-                    model="vertex_ai/gemini-2.5-flash",
+                    model=f"vertex_ai/{VERTEX_BATCH_BACKEND}",
                     vertex_project="os.environ/VERTEXAI_PROJECT",
                     vertex_location="us-central1",
                     vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
@@ -81,7 +88,7 @@ class Provider:
                 )
             case "bedrock":
                 return LiteLLMParamsBody(
-                    model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
+                    model=BEDROCK_BATCH_BACKEND,
                     aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
                     aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
                     aws_region_name="os.environ/AWS_REGION",
@@ -132,21 +139,21 @@ PROVIDERS: tuple[Provider, ...] = (
     Provider(
         "azure",
         batch_model_name("azure-batch"),
-        "gpt-5.4-mini-batch",
+        AZURE_BATCH_BACKEND,
         can_cancel=True,
         can_list=True,
     ),
     Provider(
         "vertex_ai",
         batch_model_name("vertex-batch"),
-        "gemini-2.5-flash",
+        VERTEX_BATCH_BACKEND,
         can_cancel=True,
         can_list=True,
     ),
     Provider(
         "bedrock",
         batch_model_name("bedrock-batch"),
-        "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
+        BEDROCK_BATCH_BACKEND,
         can_cancel=True,
         can_list=True,
     ),
@@ -181,6 +188,29 @@ CAPABILITIES: tuple[Capability, ...] = tuple(
 )
 
 
+def lifecycle_meta(cap: Capability) -> pytest.MarkDecorator:
+    return meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider(cap.provider),),
+            models=(cap.raw_model,),
+            mode=Mode.BATCH,
+        )
+    )
+
+
+def file_content_meta(provider: Provider) -> pytest.MarkDecorator:
+    return meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.FILES,
+            providers=(MetaProvider(provider.name),),
+            models=(provider.raw_model,),
+        )
+    )
+
+
 def raw_id_matches_provider(provider: str, batch_id: str) -> bool:
     if provider in ("openai", "azure"):
         return batch_id.startswith("batch")

tests/e2e/batches/test_batches_e2e.py

@@ -44,17 +44,22 @@ from capabilities import (
     OPENAI_BATCH_BACKEND,
     OPENAI_BATCH_MODEL,
     PROVIDERS,
+    VERTEX_BATCH_BACKEND,
     Capability,
     Provider,
     batch_model_name,
     coverage_cells_for_lifecycle,
     decoded_model_from_id,
+    file_content_meta,
     is_managed_id,
+    lifecycle_meta,
     matches_id_shape,
     openai_batch_params,
     raw_id_matches_provider,
 )
 from e2e_config import MASTER_KEY, PROXY_BASE_URL, unique_marker
+from e2e_metadata import Domain, Mode, Route, Subject, meta
+from e2e_metadata import Provider as MetaProvider
 from e2e_http import (
     FileUploadForm,
     Result,
@@ -249,7 +254,7 @@ def assert_batch_object(batch: BatchObject) -> None:
         pytest.param(
             cap,
             id=cap.id,
-            marks=pytest.mark.covers(*coverage_cells_for_lifecycle(cap)),
+            marks=(pytest.mark.covers(*coverage_cells_for_lifecycle(cap)), lifecycle_meta(cap)),
         )
         for cap in CAPABILITIES
     ],
@@ -350,6 +355,15 @@ def test_batch_lifecycle(
 
 
 @pytest.mark.covers("llm.batches.openai.key_model_access_denied.nonstream.works")
+@meta(
+    Subject(
+        domain=Domain.PROXY_AUTH,
+        route=Route.BATCHES,
+        providers=(MetaProvider.OPENAI,),
+        models=(OPENAI_BATCH_BACKEND,),
+        mode=Mode.BATCH,
+    )
+)
 def test_batch_key_model_access_denied(
     client: BatchClient, resources: ResourceManager, batch_deployments: None
 ) -> None:
@@ -389,6 +403,14 @@ def test_batch_key_model_access_denied(
     "llm.files.openai.upload.nonstream.works",
     "llm.files.openai.delete.nonstream.works",
 )
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.FILES,
+        providers=(MetaProvider.OPENAI,),
+        models=(OPENAI_BATCH_BACKEND,),
+    )
+)
 def test_file_upload_and_delete_outputs(
     client: BatchClient, resources: ResourceManager, batch_deployments: None
 ) -> None:
@@ -433,6 +455,15 @@ def unattributed_rows(rows: list[SpendLogRow]) -> list[SpendLogRow]:
         "once the fetch is bounded."
     )
 )
+@meta(
+    Subject(
+        domain=Domain.SPEND_BUDGETS,
+        route=Route.BATCHES,
+        providers=(MetaProvider.OPENAI,),
+        models=(OPENAI_BATCH_BACKEND,),
+        mode=Mode.BATCH,
+    )
+)
 def test_rate_limited_batch_create_leaves_no_unattributed_spend_row(
     client: BatchClient, resources: ResourceManager, batch_deployments: None
 ) -> None:
@@ -471,7 +502,7 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row(
 
     file = unwrap(
         client.upload_file(
-            content=render_jsonl("gpt-4o-mini"),
+            content=render_jsonl(OPENAI_BATCH_BACKEND),
             form=FileUploadForm(purpose="batch"),
             model=OPENAI_BATCH_MODEL,
             key=key,
@@ -520,6 +551,14 @@ class TestBatchFileContent:
         "llm.files.openai.content.nonstream.works",
         exercised_on=["files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.FILES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+        )
+    )
     def test_file_content_matches_upload(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -558,8 +597,9 @@ class TestBatchFileContent:
             pytest.param(
                 p,
                 id=p.name,
-                marks=pytest.mark.covers(
-                    FILE_CONTENT_CELLS[p.name], exercised_on=["files"]
+                marks=(
+                    pytest.mark.covers(FILE_CONTENT_CELLS[p.name], exercised_on=["files"]),
+                    file_content_meta(p),
                 ),
             )
             for p in PROVIDERS
@@ -632,6 +672,14 @@ class TestOpenAIFiles:
             "marker when LIT-4820 is fixed; do not relax the assertion to make it pass."
         )
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.FILES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+        )
+    )
     def test_uploaded_file_appears_in_list(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -662,6 +710,7 @@ class TestOpenAIFiles:
         "llm.files.openai.list_isolation.nonstream.works",
         exercised_on=["files"],
     )
+    @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.FILES, providers=(MetaProvider.OPENAI,)))
     def test_list_page_cursors_address_only_the_callers_own_files(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -697,6 +746,14 @@ class TestOpenAIFiles:
         "llm.files.openai.retrieve.nonstream.works",
         exercised_on=["files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.FILES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+        )
+    )
     def test_retrieve_round_trips_metadata(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -760,6 +817,15 @@ class TestBatchRateLimitErrorMapping:
         "quota_management.ratelimit.batch_rpm.blocks_over_limit",
         exercised_on=["batches"],
     )
+    @meta(
+        Subject(
+            domain=Domain.SPEND_BUDGETS,
+            route=Route.BATCHES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_batch_create_over_rpm_returns_mapped_429(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -773,7 +839,7 @@ class TestBatchRateLimitErrorMapping:
 
         file = unwrap(
             client.upload_file(
-                content=_multi_request_jsonl("gpt-4o-mini", BATCH_RL_REQUEST_LINES),
+                content=_multi_request_jsonl(OPENAI_BATCH_BACKEND, BATCH_RL_REQUEST_LINES),
                 form=FileUploadForm(purpose="batch"),
                 model=OPENAI_BATCH_MODEL,
                 key=key,
@@ -826,7 +892,7 @@ class TestBatchEnqueuedTokenLimit:
     ) -> FileObject:
         file = unwrap(
             client.upload_file(
-                content=_multi_request_jsonl("gpt-4o-mini", BATCH_RL_REQUEST_LINES),
+                content=_multi_request_jsonl(OPENAI_BATCH_BACKEND, BATCH_RL_REQUEST_LINES),
                 form=FileUploadForm(purpose="batch"),
                 model=OPENAI_BATCH_MODEL,
                 key=key,
@@ -859,6 +925,15 @@ class TestBatchEnqueuedTokenLimit:
         "quota_management.ratelimit.batch_enqueued_tokens.accepts_over_rpm",
         exercised_on=["batches"],
     )
+    @meta(
+        Subject(
+            domain=Domain.SPEND_BUDGETS,
+            route=Route.BATCHES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_enqueued_allowance_accepts_batch_over_key_rpm(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -890,6 +965,15 @@ class TestBatchEnqueuedTokenLimit:
         "quota_management.ratelimit.batch_enqueued_tokens.refunds_on_cancel",
         exercised_on=["batches"],
     )
+    @meta(
+        Subject(
+            domain=Domain.SPEND_BUDGETS,
+            route=Route.BATCHES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_exhausted_allowance_blocks_until_cancel_refunds(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -985,6 +1069,15 @@ class TestBedrockBatchAssumeRole:
         "llm.files.bedrock.upload.nonstream.works",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.BEDROCK,),
+            models=(ASSUME_ROLE_RAW_MODEL,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_unified_batch_create_with_assume_role(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -1052,6 +1145,14 @@ class TestBedrockBatchSplitS3Credentials:
         "llm.files.bedrock.split_s3_credentials.nonstream.works",
         exercised_on=["files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.FILES,
+            providers=(MetaProvider.BEDROCK,),
+            models=(ASSUME_ROLE_RAW_MODEL,),
+        )
+    )
     def test_file_lifecycle_signs_s3_with_s3_credentials(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -1123,6 +1224,15 @@ class TestBedrockBatchGovCloud:
         "llm.files.bedrock.govcloud_partition.nonstream.works",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.BEDROCK,),
+            models=(GOVCLOUD_RAW_MODEL,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_unified_file_upload_and_batch_create_in_govcloud(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -1193,6 +1303,14 @@ class TestGeminiFiles:
         "llm.files.gemini.upload.nonstream.works",
         exercised_on=["files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.FILES,
+            providers=(MetaProvider.GEMINI,),
+            models=(GEMINI_FILES_RAW_MODEL,),
+        )
+    )
     def test_gemini_file_upload(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -1227,7 +1345,7 @@ def _vllm_params(api_base: str, api_key: str | None, model_id: str) -> LiteLLMPa
     )
 
 
-HOSTED_VLLM_DEFAULT_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"
+HOSTED_VLLM_MODEL: Final = (os.environ.get("HOSTED_VLLM_MODEL") or "Qwen/Qwen2.5-0.5B-Instruct").strip()
 HOSTED_VLLM_BAD_LINE_CUSTOM_ID = "req-bad"
 
 
@@ -1236,9 +1354,8 @@ def _hosted_vllm_deployment(client: BatchClient, resources: ResourceManager) ->
     if api_base is None:
         pytest.skip("set HOSTED_VLLM_API_BASE (the live vLLM server this deployment targets)")
     api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None
-    model_id = (os.environ.get("HOSTED_VLLM_MODEL") or HOSTED_VLLM_DEFAULT_MODEL).strip()
     proxy_name = batch_model_name("hosted-vllm-batch")
-    model_row_id = client.create_model(proxy_name, _vllm_params(api_base, api_key, model_id))
+    model_row_id = client.create_model(proxy_name, _vllm_params(api_base, api_key, HOSTED_VLLM_MODEL))
     resources.defer(lambda: client.delete_model(model_row_id))
     return proxy_name
 
@@ -1290,6 +1407,15 @@ class TestHostedVllmBatch:
         "llm.files.hosted_vllm.upload.nonstream.works",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.HOSTED_VLLM,),
+            models=(HOSTED_VLLM_MODEL,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_batch_runs_to_completion_with_a_downloadable_output(
         self, client: BatchClient, resources: ResourceManager, upload_route: str
     ) -> None:
@@ -1337,6 +1463,15 @@ class TestHostedVllmBatch:
         )
 
     @pytest.mark.covers("llm.batches.hosted_vllm.basic.nonstream.works", exercised_on=["batches", "files"])
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.HOSTED_VLLM,),
+            models=(HOSTED_VLLM_MODEL,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_failing_line_lands_in_the_error_file_not_the_batch_status(
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
@@ -1417,6 +1552,7 @@ class TestBatchFailurePaths:
         "llm.batches.openai.malformed_jsonl.nonstream.works",
         exercised_on=["files"],
     )
+    @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.FILES))
     def test_malformed_jsonl_upload_rejected(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -1439,13 +1575,22 @@ class TestBatchFailurePaths:
         "llm.batches.openai.cancel_terminal.nonstream.works",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_endpoint_mismatch_fails_batch_and_cancel_conflicts(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
         key = resources.key()
         file = unwrap(
             client.upload_file(
-                content=_mismatched_endpoint_jsonl("gpt-4o-mini"),
+                content=_mismatched_endpoint_jsonl(OPENAI_BATCH_BACKEND),
                 form=FileUploadForm(purpose="batch"),
                 model=OPENAI_BATCH_MODEL,
                 key=key,
@@ -1495,6 +1640,15 @@ class TestBatchFailurePaths:
         "llm.batches.openai.foreign_file_id.nonstream.works",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.AZURE,),
+            models=(AZURE_BATCH_RAW_MODEL,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_foreign_encoded_file_id_routes_by_file_model(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -1544,6 +1698,15 @@ class TestBatchSecondHop:
         "llm.batches.openai.second_hop.nonstream.works",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.LITELLM_PROXY, MetaProvider.OPENAI),
+            models=(OPENAI_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_unified_create_and_retrieve_via_chained_gateway(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -1561,7 +1724,7 @@ class TestBatchSecondHop:
 
         file = unwrap(
             client.upload_file(
-                content=render_jsonl("gpt-4o-mini"),
+                content=render_jsonl(OPENAI_BATCH_BACKEND),
                 form=FileUploadForm(purpose="batch", target_model_names=hop_name),
                 key=key,
             )
@@ -1680,13 +1843,22 @@ class TestBatchTerminalState:
         "llm.batches.openai.terminal_state.nonstream.cost_logged",
         exercised_on=["batches", "files"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.OPENAI,),
+            models=(OPENAI_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_completed_batch_downloads_output_and_books_cost(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
         key = resources.key()
         file = unwrap(
             client.upload_file(
-                content=render_jsonl("gpt-4o-mini"),
+                content=render_jsonl(OPENAI_BATCH_BACKEND),
                 form=FileUploadForm(purpose="batch"),
                 model=OPENAI_BATCH_MODEL,
                 key=key,
@@ -1786,6 +1958,15 @@ class TestVertexNativePassthrough:
         "llm.batches.vertex.native_passthrough.nonstream.works",
         exercised_on=["files", "batches"],
     )
+    @meta(
+        Subject(
+            domain=Domain.LLM_TRANSLATION,
+            route=Route.BATCHES,
+            providers=(MetaProvider.VERTEX_AI,),
+            models=(VERTEX_BATCH_BACKEND,),
+            mode=Mode.BATCH,
+        )
+    )
     def test_native_jsonl_round_trips_untouched_and_starts_a_batch(
         self, client: BatchClient, resources: ResourceManager, batch_deployments: None
     ) -> None:
@@ -1848,6 +2029,7 @@ class TestVertexNativePassthrough:
             ),
         ],
     )
+    @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.FILES))
     def test_passthrough_upload_is_rejected_outside_a_native_vertex_batch(
         self,
         content: bytes,

tests/e2e/batches/test_managed_files_enforcement_e2e.py

@@ -22,9 +22,10 @@ import pytest
 
 from batch_client import BatchClient, FileObject
 from batch_cleanup import cleanup_file
-from capabilities import batch_model_name, is_managed_id, openai_batch_params
+from capabilities import OPENAI_BATCH_BACKEND, batch_model_name, is_managed_id, openai_batch_params
 from e2e_config import unique_marker
 from e2e_http import FileUploadForm, Result, UnknownApiError, unwrap
+from e2e_metadata import Domain, Provider, Route, Subject, meta
 from lifecycle import ResourceManager
 
 pytestmark = [pytest.mark.e2e, pytest.mark.managed_files]
@@ -64,6 +65,12 @@ def managed_model(client: BatchClient) -> Iterator[str]:
 
 
 @pytest.mark.covers(UPLOAD_ROW)
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.FILES,
+    )
+)
 def test_upload_without_target_model_names_rejected(
     client: BatchClient, scoped_key: str, managed_model: str
 ) -> None:
@@ -76,6 +83,12 @@ def test_upload_without_target_model_names_rejected(
 
 
 @pytest.mark.covers(UPLOAD_ROW)
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.FILES,
+    )
+)
 def test_upload_with_model_param_rejected(
     client: BatchClient, scoped_key: str, managed_model: str
 ) -> None:
@@ -89,12 +102,26 @@ def test_upload_with_model_param_rejected(
 
 
 @pytest.mark.covers(ISOLATION_ROW)
+@meta(
+    Subject(
+        domain=Domain.PROXY_AUTH,
+        route=Route.FILES,
+    )
+)
 def test_raw_provider_file_id_rejected(client: BatchClient, scoped_key: str) -> None:
     result = client.retrieve_file("file-e2e-raw-provider-id", key=scoped_key)
     expect_api_error(result, 400, "Raw provider file ids cannot be used")
 
 
 @pytest.mark.covers(ISOLATION_ROW)
+@meta(
+    Subject(
+        domain=Domain.PROXY_AUTH,
+        route=Route.FILES,
+        providers=(Provider.OPENAI,),
+        models=(OPENAI_BATCH_BACKEND,),
+    )
+)
 def test_cross_user_managed_id_denied_owner_allowed(
     client: BatchClient, resources: ResourceManager, managed_model: str
 ) -> None:

tests/e2e/claude_code/_basic_messaging.py

@@ -31,6 +31,8 @@ from typing import Any, Callable, Mapping, Sequence
 
 import pytest
 
+from e2e_metadata import step
+
 from claude_code._env import require_proxy
 from claude_code.cli_driver import (
     ClaudeCLIError,
@@ -74,6 +76,7 @@ def _count_stream_event_deltas(events: Sequence[Mapping[str, Any]]) -> int:
     return count
 
 
+@step("Run Claude Code headless against {models} through the proxy and check every model replies")
 def run_basic_messaging_cell(
     *,
     compat_result,

tests/e2e/claude_code/_passthrough.py

@@ -58,6 +58,8 @@ from typing import Any, Callable, Dict, Mapping, Optional, Sequence
 
 import pytest
 
+from e2e_metadata import step
+
 from claude_code._env import require_proxy
 from claude_code.cli_driver import (
     ClaudeCLIError,
@@ -118,6 +120,10 @@ def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]:
     }
 
 
+@step(
+    "Run Claude Code headless against {models} through the proxy's native provider passthrough route"
+    " and check every model replies"
+)
 def run_passthrough_cell(
     *,
     compat_result,

tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py

@@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 # Per the PRD: each cell is exercised against three Claude tiers via the
@@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.ANTHROPIC,),
+        models=tuple(ANTHROPIC_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
 

tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py

@@ -26,6 +26,7 @@ the matrix builder still sees three rows for this (feature, provider).
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 # Per-model aliases registered in the LiteLLM proxy's routing config to
@@ -40,6 +41,15 @@ AZURE_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.AZURE_AI,),
+        models=tuple(AZURE_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
 

tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py

@@ -25,6 +25,7 @@ green if all three pass.
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 AZURE_OPENAI_MODELS = [
@@ -34,6 +35,15 @@ AZURE_OPENAI_MODELS = [
 ]
 
 
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.AZURE,),
+        models=tuple(AZURE_OPENAI_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_azure_openai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty reply from each GPT-5.6 tier."""

tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py

@@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 # Per-model aliases registered in the LiteLLM proxy's routing config to
@@ -35,6 +36,15 @@ BEDROCK_CONVERSE_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.BEDROCK,),
+        models=tuple(BEDROCK_CONVERSE_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
     run_basic_messaging_cell(

tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py

@@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 # Per-model aliases registered in the LiteLLM proxy's routing config to
@@ -35,6 +36,15 @@ BEDROCK_INVOKE_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.bedrock_invoke.basic.nonstream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.BEDROCK,),
+        models=tuple(BEDROCK_INVOKE_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
     run_basic_messaging_cell(

tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py

@@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
 
@@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [
 ]
 
 
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.BEDROCK_MANTLE,),
+        models=tuple(BEDROCK_MANTLE_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_bedrock_mantle(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty reply from each GPT-5.6 tier."""

tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py

@@ -22,6 +22,7 @@ green if all three pass.
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
 
@@ -32,6 +33,15 @@ OPENAI_MODELS = [
 ]
 
 
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.OPENAI,),
+        models=tuple(OPENAI_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_openai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty reply from each GPT-5.6 tier."""

tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py

@@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 # Per-model aliases registered in the LiteLLM proxy's routing config to
@@ -35,6 +36,15 @@ VERTEX_AI_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.vertex.basic.nonstream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.VERTEX_AI,),
+        models=tuple(VERTEX_AI_MODELS),
+        mode=Mode.NONSTREAM,
+    )
+)
 def test_basic_messaging_non_streaming_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
     run_basic_messaging_cell(

tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py

@@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Subject, meta
 from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON
 
 
+@meta(Subject(domain=Domain.LLM_TRANSLATION))
 def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result):
     """Record the static not_applicable outcome for this cell."""
     compat_result.set(

tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py

@@ -26,6 +26,7 @@ sees three rows for this (feature, provider).
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 ANTHROPIC_MODELS = [
@@ -36,6 +37,15 @@ ANTHROPIC_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.anthropic.basic.stream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.ANTHROPIC,),
+        models=tuple(ANTHROPIC_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (one row per Claude tier).

tests/e2e/claude_code/basic_messaging_streaming/test_azure.py

@@ -20,6 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 AZURE_MODELS = [
@@ -30,6 +31,15 @@ AZURE_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.AZURE_AI,),
+        models=tuple(AZURE_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (one row per Claude tier).

tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py

@@ -24,6 +24,7 @@ green if all three pass.
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 AZURE_OPENAI_MODELS = [
@@ -33,6 +34,15 @@ AZURE_OPENAI_MODELS = [
 ]
 
 
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.AZURE,),
+        models=tuple(AZURE_OPENAI_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_azure_openai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply from each GPT-5.6 tier."""

tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py

@@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 BEDROCK_CONVERSE_MODELS = [
@@ -26,6 +27,15 @@ BEDROCK_CONVERSE_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.BEDROCK,),
+        models=tuple(BEDROCK_CONVERSE_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (one row per Claude tier).

tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py

@@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 BEDROCK_INVOKE_MODELS = [
@@ -26,6 +27,15 @@ BEDROCK_INVOKE_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.bedrock_invoke.basic.stream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.BEDROCK,),
+        models=tuple(BEDROCK_INVOKE_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (one row per Claude tier).

tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py

@@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`).
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 from claude_code._gpt_cells import skip_unless_mantle_cells_enabled
 
@@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [
 ]
 
 
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.BEDROCK_MANTLE,),
+        models=tuple(BEDROCK_MANTLE_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_bedrock_mantle(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply from each GPT-5.6 tier."""

tests/e2e/claude_code/basic_messaging_streaming/test_openai.py

@@ -24,6 +24,7 @@ green if all three pass.
 
 from __future__ import annotations
 
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled
 
@@ -34,6 +35,15 @@ OPENAI_MODELS = [
 ]
 
 
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.OPENAI,),
+        models=tuple(OPENAI_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_openai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply from each GPT-5.6 tier."""

tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py

@@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
 from __future__ import annotations
 
 import pytest
+from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from claude_code._basic_messaging import run_basic_messaging_cell
 
 VERTEX_AI_MODELS = [
@@ -26,6 +27,15 @@ VERTEX_AI_MODELS = [
 
 
 @pytest.mark.covers("llm.messages.vertex.basic.stream.works")
+@meta(
+    Subject(
+        domain=Domain.LLM_TRANSLATION,
+        route=Route.MESSAGES,
+        providers=(Provider.VERTEX_AI,),
+        models=tuple(VERTEX_AI_MODELS),
+        mode=Mode.STREAM,
+    )
+)
 def test_basic_messaging_streaming_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (one row per Claude tier).