litellm/tests/proxy_behavior/lens/feedback_cases.json

{
  "checks": [
    {
      "id": "completion",
      "instruction": "Did the agent deliver the requested answer or artifact? Distinguish a missing recorded answer from evidence that the task was not completed.",
      "enabled": true
    },
    {
      "id": "handoff",
      "instruction": "Did required handoffs actually reach the next agent? Normal handoff control flow and successful recovery are not failures.",
      "enabled": true
    },
    {
      "id": "research_quality",
      "instruction": "Do final claims match retrieved evidence? Identify concrete unsupported or contradicted conclusions, not hypothetical missing research topics.",
      "enabled": true
    },
    {
      "id": "efficiency",
      "instruction": "Identify repeated work that produced no additional information. Do not mistake retrying a failed operation for redundant successful work.",
      "enabled": true
    },
    {
      "id": "observability",
      "instruction": "Identify gaps in recorded task, output, or workflow evidence that prevent a reliable assessment. Do not equate incomplete logs with task failure.",
      "enabled": true
    },
    {
      "id": "tool_failure",
      "instruction": "Identify unrecovered tool failures that block the requested task. Recovered errors are successful patterns, not unresolved failures.",
      "enabled": true
    }
  ],
  "feedback": [
    {
      "id": "accepted-probes",
      "check_id": "efficiency",
      "title": "Repeated successful cache lookups",
      "description": "Several successful cache lookups returned the same document.",
      "kind": "issue",
      "status": "dismissed",
      "reason": "Up to four identical successful cache reads are intentional freshness probes in this workflow. More than four redundant reads is still wasteful. This exception does not excuse failed tools, incomplete answers, or unsupported claims.",
      "evidence": [
        {
          "execution_id": "previous-run",
          "span_id": "cache",
          "quote": "Identical cached document."
        }
      ],
      "first_seen": "2026-01-01T00:00:00Z",
      "last_seen": "2026-01-01T00:00:00Z",
      "revision": 1
    }
  ],
  "cases": [
    {
      "name": "accepted_cache_probes",
      "split": "holdout",
      "task": "Find the published release date.",
      "answer": "The release date is June 12.",
      "steps": [
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ]
      ],
      "expected": [],
      "context": "An assistant must complete the user request and support factual claims with tool evidence. No specific agent roles are mandatory unless the task says so.",
      "missing_root": false,
      "incomplete": false
    },
    {
      "name": "excessive_cache_probes",
      "split": "holdout",
      "task": "Find the published release date.",
      "answer": "The release date is June 12.",
      "steps": [
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ],
        [
          "search",
          "tool",
          "release date",
          "Release date: June 12. Identical cached document.",
          "OK"
        ]
      ],
      "expected": [
        "efficiency"
      ],
      "context": "An assistant must complete the user request and support factual claims with tool evidence. No specific agent roles are mandatory unless the task says so.",
      "missing_root": false,
      "incomplete": false
    },
    {
      "name": "contradicted_claim",
      "split": "holdout",
      "task": "What were June sales?",
      "answer": "June sales were 250 units.",
      "steps": [
        [
          "sales_record",
          "tool",
          "June",
          "June sales were 125 units.",
          "OK"
        ]
      ],
      "expected": [
        "research_quality"
      ],
      "context": "An assistant must complete the user request and support factual claims with tool evidence. No specific agent roles are mandatory unless the task says so.",
      "missing_root": false,
      "incomplete": false
    }
  ]
}