From 3c9f6fe492a7383ff06c05567bd04616b6f90f65 Mon Sep 17 00:00:00 2001 From: Bethel Yohannes Date: Sat, 21 Feb 2026 16:11:49 +0300 Subject: [PATCH] created test scenrio and tested it for semntic tracking ledger --- .orchestration/agent_trace.jsonl | 8 ++- docs/testing-traceability.md | 78 +++++++++++++++++++++ package.json | 2 + scripts/test-hooks.ts | 17 +++++ scripts/test-trace-scenario.ts | 114 +++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 docs/testing-traceability.md create mode 100644 scripts/test-trace-scenario.ts diff --git a/.orchestration/agent_trace.jsonl b/.orchestration/agent_trace.jsonl index 297c6c1735..75a357fc26 100644 --- a/.orchestration/agent_trace.jsonl +++ b/.orchestration/agent_trace.jsonl @@ -1,3 +1,5 @@ -{"id":"02a7c124-4730-4e49-8783-08f861f640ba","timestamp":"2026-02-21T09:48:48.707Z","vcs":{"revision_id":"abc123"},"files":[{"relative_path":"src/api/weather.ts","conversations":[{"url":"session-1","contributor":{"entity_type":"AI","model_identifier":"test-model"},"ranges":[{"start_line":1,"end_line":2,"content_hash":"sha256:2e4ec5cf14cfeb55ef32cb663ffd2c4fe6ab22bb2d2fe77ee8e284fe62866cac"}],"related":[{"type":"specification","value":"INT-001"}]}]}]} -{"id":"e55e9867-663a-412e-b341-af61f85dbedd","timestamp":"2026-02-21T09:48:48.716Z","files":[{"relative_path":"src/api/forecast.ts","conversations":[{"contributor":{"entity_type":"AI","model_identifier":"unknown"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:c47272be41ffa9d8a897b81b3f9f6d5e13fbb5d6dd4609a7b20c282950f11842"}],"related":[{"type":"specification","value":"INT-001"}]}]}]} -{"id":"3609a24b-418e-4e1a-a554-2e3cf9db1be9","timestamp":"2026-02-21T09:48:48.737Z","vcs":{"revision_id":"rev-1"},"files":[{"relative_path":"src/api/demo.ts","conversations":[{"url":"log-1","contributor":{"entity_type":"AI","model_identifier":"model-1"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:bd994c26b795571c7c1e0357cba348f1d0238b648b748b9ec7a4d0de264d212f"}],"related":[{"type":"specification","value":"INT-001"}]}]}]} +{"id":"2e408f80-f271-4f66-a728-76117c9301ca","timestamp":"2026-02-21T13:09:40.095Z","vcs":{"revision_id":"abc123"},"files":[{"relative_path":"src/api/weather.ts","conversations":[{"url":"session-1","contributor":{"entity_type":"AI","model_identifier":"test-model"},"ranges":[{"start_line":1,"end_line":2,"content_hash":"sha256:2e4ec5cf14cfeb55ef32cb663ffd2c4fe6ab22bb2d2fe77ee8e284fe62866cac"}],"related":[{"type":"specification","value":"INT-001"}]}]}]} +{"id":"58f39b6a-cb85-4a08-9f17-c41db429f9f7","timestamp":"2026-02-21T13:09:40.116Z","files":[{"relative_path":"src/api/forecast.ts","conversations":[{"contributor":{"entity_type":"AI","model_identifier":"unknown"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:c47272be41ffa9d8a897b81b3f9f6d5e13fbb5d6dd4609a7b20c282950f11842"}],"related":[{"type":"specification","value":"INT-001"}]}]}]} +{"id":"adcb313c-4db0-4168-bf96-b3e9df9a3346","timestamp":"2026-02-21T13:09:40.120Z","files":[{"relative_path":"src/api/trace.ts","conversations":[{"contributor":{"entity_type":"AI","model_identifier":"unknown"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:a3d4bb671e8b19ed161b408c183193afc9fa361f573c5187dded809930d14413"}],"related":[{"type":"specification","value":"INT-001"},{"type":"request","value":"REQ-12345"}]}]}]} +{"id":"52e92ed0-7aff-490e-9b3a-f3704926e858","timestamp":"2026-02-21T13:09:40.140Z","vcs":{"revision_id":"rev-1"},"files":[{"relative_path":"src/api/demo.ts","conversations":[{"url":"log-1","contributor":{"entity_type":"AI","model_identifier":"model-1"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:bd994c26b795571c7c1e0357cba348f1d0238b648b748b9ec7a4d0de264d212f"}],"related":[{"type":"specification","value":"INT-001"}]}]}]} +{"id":"35b7b5a9-263a-48a0-bad0-89743ebf0368","timestamp":"2026-02-21T13:10:04.649Z","files":[{"relative_path":"src/api/weather.ts","conversations":[{"url":"session-scenario","contributor":{"entity_type":"AI","model_identifier":"scenario-runner"},"ranges":[{"start_line":1,"end_line":4,"content_hash":"sha256:6ecdce6218d02c1a1f3864f4a0e7829f1c0887e3f5a9c57d77331e473841c30c"}],"related":[{"type":"specification","value":"INT-001"},{"type":"request","value":"REQ-scenario-1771679404634"}]}]}]} diff --git a/docs/testing-traceability.md b/docs/testing-traceability.md new file mode 100644 index 0000000000..5c1f4f671c --- /dev/null +++ b/docs/testing-traceability.md @@ -0,0 +1,78 @@ +# Testing the AI-Native Git Layer (Traceability) + +How to test the semantic tracking ledger and `agent_trace.jsonl` **in the terminal**. + +## Prerequisites + +- From the **repo root**: `.orchestration/active_intents.yaml` must exist (it does in this repo). +- Node/pnpm: `pnpm install` already run. + +## 1. Run the full hook test suite + +Runs content-hash, context-loader, scope, pre-hook, post-hook, and middleware: + +```bash +pnpm test:hooks +``` + +Or without pnpm: + +```bash +npx tsx scripts/test-hooks.ts +``` + +## 2. Run the trace scenario (one flow end-to-end) + +Simulates: **select_active_intent** → **write_to_file** (pre-hook) → **post-hook** appends to `agent_trace.jsonl`, then prints the last trace entry. + +```bash +pnpm test:trace-scenario +``` + +Or: + +```bash +npx tsx scripts/test-trace-scenario.ts +``` + +You should see: + +- Intent `INT-001` loaded +- `write_to_file` allowed (in scope) +- A new line in `.orchestration/agent_trace.jsonl` +- The last entry printed with `intent_id`, `content_hash`, and **REQ-ID** in `related` + +## 3. Inspect the trace file yourself + +After running the scenario (or after real agent writes): + +```bash +# Last 3 trace entries (pretty-printed) +tail -n 3 .orchestration/agent_trace.jsonl | while read line; do echo "$line" | jq .; done + +# Or just the raw last line +tail -n 1 .orchestration/agent_trace.jsonl +``` + +## 4. Create your own scenario + +Copy `scripts/test-trace-scenario.ts` and change: + +- `intent_id` (must exist in `.orchestration/active_intents.yaml`) +- `path` (must be under that intent’s `owned_scope`, e.g. `src/api/**`) +- `content` and `mutation_class` (`AST_REFACTOR` | `INTENT_EVOLUTION` | `NEW_FILE`) + +Then run: + +```bash +pnpm exec tsx scripts/your-scenario.ts +``` + +## Summary + +| Command | What it does | +| -------------------------- | ----------------------------------------------------------------------- | +| `pnpm test:hooks` | All hook unit-style checks (content-hash, pre/post-hook, middleware). | +| `pnpm test:trace-scenario` | One full flow: select intent → write → append trace → print last entry. | + +Both run in the terminal with no VS Code or extension required. diff --git a/package.json b/package.json index de8dff751c..5043923f7e 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "lint": "turbo lint --log-order grouped --output-logs new-only", "check-types": "turbo check-types --log-order grouped --output-logs new-only", "test": "turbo test --log-order grouped --output-logs new-only", + "test:hooks": "pnpm exec tsx scripts/test-hooks.ts", + "test:trace-scenario": "pnpm exec tsx scripts/test-trace-scenario.ts", "format": "turbo format --log-order grouped --output-logs new-only", "build": "turbo build --log-order grouped --output-logs new-only", "bundle": "turbo bundle --log-order grouped --output-logs new-only", diff --git a/scripts/test-hooks.ts b/scripts/test-hooks.ts index f7176b69a5..545fe86307 100644 --- a/scripts/test-hooks.ts +++ b/scripts/test-hooks.ts @@ -215,6 +215,23 @@ async function main() { assert(lines.length >= 2, "two or more lines") }) + await run("appendAgentTrace injects REQ-ID into related array", async () => { + await appendAgentTrace(cwd, { + relativePath: "src/api/trace.ts", + content: "// trace", + intentId: "INT-001", + reqId: "REQ-12345", + }) + const raw = await fs.readFile(tracePath, "utf-8") + const lines = raw.trim().split("\n").filter(Boolean) + const lastLine = lines[lines.length - 1] + assert(lastLine != null, "last line exists") + const entry = JSON.parse(lastLine) + const conv = entry.files[0].conversations[0] + const reqRelated = conv.related?.find((r: { type: string }) => r.type === "request") + assert(reqRelated != null && reqRelated.value === "REQ-12345", "REQ-ID in related array") + }) + console.log("\n=== 6. middleware (full flow) ===\n") activeIntentId = null // reset so we simulate: select_intent then write diff --git a/scripts/test-trace-scenario.ts b/scripts/test-trace-scenario.ts new file mode 100644 index 0000000000..9e9e5b3f6a --- /dev/null +++ b/scripts/test-trace-scenario.ts @@ -0,0 +1,114 @@ +#!/usr/bin/env npx tsx +/** + * Scenario test for the AI-Native Git Layer (agent_trace.jsonl). + * + * Run from repo root: + * pnpm tsx scripts/test-trace-scenario.ts + * or + * npx tsx scripts/test-trace-scenario.ts + * + * This script simulates: select_active_intent → write_to_file → post-hook appends to agent_trace.jsonl. + * It then prints the last trace entry so you can verify intent_id, mutation_class, content_hash, and REQ-ID. + */ +import fs from "fs/promises" +import path from "path" + +import { PreHook } from "../src/hooks/pre-hook" +import { HookMiddleware } from "../src/hooks/middleware" +import { contentHash } from "../src/hooks/content-hash" + +const cwd = process.cwd() +const tracePath = path.join(cwd, ".orchestration", "agent_trace.jsonl") + +async function main() { + console.log("=== Traceability scenario (terminal test) ===\n") + + let activeIntentId: string | null = null + const preHook = new PreHook({ + cwd, + getActiveIntentId: () => activeIntentId, + setActiveIntentId: (id) => { + activeIntentId = id + }, + requireIntentForDestructiveOnly: true, + }) + + const middleware = new HookMiddleware({ + preHook, + getActiveIntentId: () => activeIntentId, + getCwd: () => cwd, + getReqId: () => "REQ-scenario-" + Date.now(), + getSessionLogId: () => "session-scenario", + getModelId: () => "scenario-runner", + getVcsRevisionId: () => undefined, + }) + + // Step 1: Select intent (like the agent would) + console.log("1. select_active_intent(INT-001) ...") + const selectResult = await middleware.preToolUse("select_active_intent", { intent_id: "INT-001" }) + if (selectResult.blocked) { + console.error(" FAIL: select_active_intent blocked:", selectResult.error) + process.exit(1) + } + console.log(" OK – intent context loaded\n") + + // Step 2: Simulate write_to_file with intent_id and mutation_class + const writeParams = { + path: "src/api/weather.ts", + content: "// Weather API\nconst getWeather = () => ({ temp: 72 })\nexport { getWeather }\n", + intent_id: "INT-001", + mutation_class: "INTENT_EVOLUTION" as const, + } + console.log("2. write_to_file (pre-hook check) ...") + const preWrite = await middleware.preToolUse("write_to_file", writeParams) + if (preWrite.blocked) { + console.error(" FAIL: write_to_file blocked:", preWrite.error) + process.exit(1) + } + console.log(" OK – within scope\n") + + // Step 3: Post-hook appends to agent_trace.jsonl (like after real write) + console.log("3. postToolUse (append to agent_trace.jsonl) ...") + await middleware.postToolUse("write_to_file", writeParams, {}) + console.log(" OK – trace entry appended\n") + + // Step 4: Read and display the last trace entry + const raw = await fs.readFile(tracePath, "utf-8") + const lines = raw.trim().split("\n").filter(Boolean) + const lastLine = lines[lines.length - 1] + if (!lastLine) { + console.error(" FAIL: no lines in agent_trace.jsonl") + process.exit(1) + } + + const entry = JSON.parse(lastLine) + const expectedHash = contentHash(writeParams.content) + + console.log("4. Last trace entry (from .orchestration/agent_trace.jsonl):\n") + console.log(JSON.stringify(entry, null, 2)) + console.log("\n--- Checks ---") + console.log( + " intent (specification):", + entry.files[0].conversations[0].related?.find((r: { type: string }) => r.type === "specification")?.value ?? + "missing", + ) + console.log( + " REQ-ID (request): ", + entry.files[0].conversations[0].related?.find((r: { type: string }) => r.type === "request")?.value ?? + "missing", + ) + console.log( + " content_hash in ranges: ", + entry.files[0].conversations[0].ranges[0]?.content_hash?.startsWith("sha256:") ? "yes" : "no", + ) + console.log( + " expected hash match: ", + entry.files[0].conversations[0].ranges[0]?.content_hash === expectedHash ? "yes" : "no", + ) + console.log("\n=== Scenario done. ===\n") +} + +main().catch((err) => { + console.error(err) + process.exit(1) +})