created test scenrio and tested it for semntic tracking ledger

This commit is contained in:
Bethel Yohannes 2026-02-21 16:11:49 +03:00
parent e226d1e2a5
commit 3c9f6fe492
5 changed files with 216 additions and 3 deletions

View file

@ -1,3 +1,5 @@
{"id":"02a7c124-4730-4e49-8783-08f861f640ba","timestamp":"2026-02-21T09:48:48.707Z","vcs":{"revision_id":"abc123"},"files":[{"relative_path":"src/api/weather.ts","conversations":[{"url":"session-1","contributor":{"entity_type":"AI","model_identifier":"test-model"},"ranges":[{"start_line":1,"end_line":2,"content_hash":"sha256:2e4ec5cf14cfeb55ef32cb663ffd2c4fe6ab22bb2d2fe77ee8e284fe62866cac"}],"related":[{"type":"specification","value":"INT-001"}]}]}]}
{"id":"e55e9867-663a-412e-b341-af61f85dbedd","timestamp":"2026-02-21T09:48:48.716Z","files":[{"relative_path":"src/api/forecast.ts","conversations":[{"contributor":{"entity_type":"AI","model_identifier":"unknown"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:c47272be41ffa9d8a897b81b3f9f6d5e13fbb5d6dd4609a7b20c282950f11842"}],"related":[{"type":"specification","value":"INT-001"}]}]}]}
{"id":"3609a24b-418e-4e1a-a554-2e3cf9db1be9","timestamp":"2026-02-21T09:48:48.737Z","vcs":{"revision_id":"rev-1"},"files":[{"relative_path":"src/api/demo.ts","conversations":[{"url":"log-1","contributor":{"entity_type":"AI","model_identifier":"model-1"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:bd994c26b795571c7c1e0357cba348f1d0238b648b748b9ec7a4d0de264d212f"}],"related":[{"type":"specification","value":"INT-001"}]}]}]}
{"id":"2e408f80-f271-4f66-a728-76117c9301ca","timestamp":"2026-02-21T13:09:40.095Z","vcs":{"revision_id":"abc123"},"files":[{"relative_path":"src/api/weather.ts","conversations":[{"url":"session-1","contributor":{"entity_type":"AI","model_identifier":"test-model"},"ranges":[{"start_line":1,"end_line":2,"content_hash":"sha256:2e4ec5cf14cfeb55ef32cb663ffd2c4fe6ab22bb2d2fe77ee8e284fe62866cac"}],"related":[{"type":"specification","value":"INT-001"}]}]}]}
{"id":"58f39b6a-cb85-4a08-9f17-c41db429f9f7","timestamp":"2026-02-21T13:09:40.116Z","files":[{"relative_path":"src/api/forecast.ts","conversations":[{"contributor":{"entity_type":"AI","model_identifier":"unknown"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:c47272be41ffa9d8a897b81b3f9f6d5e13fbb5d6dd4609a7b20c282950f11842"}],"related":[{"type":"specification","value":"INT-001"}]}]}]}
{"id":"adcb313c-4db0-4168-bf96-b3e9df9a3346","timestamp":"2026-02-21T13:09:40.120Z","files":[{"relative_path":"src/api/trace.ts","conversations":[{"contributor":{"entity_type":"AI","model_identifier":"unknown"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:a3d4bb671e8b19ed161b408c183193afc9fa361f573c5187dded809930d14413"}],"related":[{"type":"specification","value":"INT-001"},{"type":"request","value":"REQ-12345"}]}]}]}
{"id":"52e92ed0-7aff-490e-9b3a-f3704926e858","timestamp":"2026-02-21T13:09:40.140Z","vcs":{"revision_id":"rev-1"},"files":[{"relative_path":"src/api/demo.ts","conversations":[{"url":"log-1","contributor":{"entity_type":"AI","model_identifier":"model-1"},"ranges":[{"start_line":1,"end_line":1,"content_hash":"sha256:bd994c26b795571c7c1e0357cba348f1d0238b648b748b9ec7a4d0de264d212f"}],"related":[{"type":"specification","value":"INT-001"}]}]}]}
{"id":"35b7b5a9-263a-48a0-bad0-89743ebf0368","timestamp":"2026-02-21T13:10:04.649Z","files":[{"relative_path":"src/api/weather.ts","conversations":[{"url":"session-scenario","contributor":{"entity_type":"AI","model_identifier":"scenario-runner"},"ranges":[{"start_line":1,"end_line":4,"content_hash":"sha256:6ecdce6218d02c1a1f3864f4a0e7829f1c0887e3f5a9c57d77331e473841c30c"}],"related":[{"type":"specification","value":"INT-001"},{"type":"request","value":"REQ-scenario-1771679404634"}]}]}]}

View file

@ -0,0 +1,78 @@
# Testing the AI-Native Git Layer (Traceability)
How to test the semantic tracking ledger and `agent_trace.jsonl` **in the terminal**.
## Prerequisites
- From the **repo root**: `.orchestration/active_intents.yaml` must exist (it does in this repo).
- Node/pnpm: `pnpm install` already run.
## 1. Run the full hook test suite
Runs content-hash, context-loader, scope, pre-hook, post-hook, and middleware:
```bash
pnpm test:hooks
```
Or without pnpm:
```bash
npx tsx scripts/test-hooks.ts
```
## 2. Run the trace scenario (one flow end-to-end)
Simulates: **select_active_intent****write_to_file** (pre-hook) → **post-hook** appends to `agent_trace.jsonl`, then prints the last trace entry.
```bash
pnpm test:trace-scenario
```
Or:
```bash
npx tsx scripts/test-trace-scenario.ts
```
You should see:
- Intent `INT-001` loaded
- `write_to_file` allowed (in scope)
- A new line in `.orchestration/agent_trace.jsonl`
- The last entry printed with `intent_id`, `content_hash`, and **REQ-ID** in `related`
## 3. Inspect the trace file yourself
After running the scenario (or after real agent writes):
```bash
# Last 3 trace entries (pretty-printed)
tail -n 3 .orchestration/agent_trace.jsonl | while read line; do echo "$line" | jq .; done
# Or just the raw last line
tail -n 1 .orchestration/agent_trace.jsonl
```
## 4. Create your own scenario
Copy `scripts/test-trace-scenario.ts` and change:
- `intent_id` (must exist in `.orchestration/active_intents.yaml`)
- `path` (must be under that intents `owned_scope`, e.g. `src/api/**`)
- `content` and `mutation_class` (`AST_REFACTOR` | `INTENT_EVOLUTION` | `NEW_FILE`)
Then run:
```bash
pnpm exec tsx scripts/your-scenario.ts
```
## Summary
| Command | What it does |
| -------------------------- | ----------------------------------------------------------------------- |
| `pnpm test:hooks` | All hook unit-style checks (content-hash, pre/post-hook, middleware). |
| `pnpm test:trace-scenario` | One full flow: select intent → write → append trace → print last entry. |
Both run in the terminal with no VS Code or extension required.

View file

@ -12,6 +12,8 @@
"lint": "turbo lint --log-order grouped --output-logs new-only",
"check-types": "turbo check-types --log-order grouped --output-logs new-only",
"test": "turbo test --log-order grouped --output-logs new-only",
"test:hooks": "pnpm exec tsx scripts/test-hooks.ts",
"test:trace-scenario": "pnpm exec tsx scripts/test-trace-scenario.ts",
"format": "turbo format --log-order grouped --output-logs new-only",
"build": "turbo build --log-order grouped --output-logs new-only",
"bundle": "turbo bundle --log-order grouped --output-logs new-only",

View file

@ -215,6 +215,23 @@ async function main() {
assert(lines.length >= 2, "two or more lines")
})
await run("appendAgentTrace injects REQ-ID into related array", async () => {
await appendAgentTrace(cwd, {
relativePath: "src/api/trace.ts",
content: "// trace",
intentId: "INT-001",
reqId: "REQ-12345",
})
const raw = await fs.readFile(tracePath, "utf-8")
const lines = raw.trim().split("\n").filter(Boolean)
const lastLine = lines[lines.length - 1]
assert(lastLine != null, "last line exists")
const entry = JSON.parse(lastLine)
const conv = entry.files[0].conversations[0]
const reqRelated = conv.related?.find((r: { type: string }) => r.type === "request")
assert(reqRelated != null && reqRelated.value === "REQ-12345", "REQ-ID in related array")
})
console.log("\n=== 6. middleware (full flow) ===\n")
activeIntentId = null // reset so we simulate: select_intent then write

View file

@ -0,0 +1,114 @@
#!/usr/bin/env npx tsx
/**
* Scenario test for the AI-Native Git Layer (agent_trace.jsonl).
*
* Run from repo root:
* pnpm tsx scripts/test-trace-scenario.ts
* or
* npx tsx scripts/test-trace-scenario.ts
*
* This script simulates: select_active_intent write_to_file post-hook appends to agent_trace.jsonl.
* It then prints the last trace entry so you can verify intent_id, mutation_class, content_hash, and REQ-ID.
*/
import fs from "fs/promises"
import path from "path"
import { PreHook } from "../src/hooks/pre-hook"
import { HookMiddleware } from "../src/hooks/middleware"
import { contentHash } from "../src/hooks/content-hash"
const cwd = process.cwd()
const tracePath = path.join(cwd, ".orchestration", "agent_trace.jsonl")
async function main() {
console.log("=== Traceability scenario (terminal test) ===\n")
let activeIntentId: string | null = null
const preHook = new PreHook({
cwd,
getActiveIntentId: () => activeIntentId,
setActiveIntentId: (id) => {
activeIntentId = id
},
requireIntentForDestructiveOnly: true,
})
const middleware = new HookMiddleware({
preHook,
getActiveIntentId: () => activeIntentId,
getCwd: () => cwd,
getReqId: () => "REQ-scenario-" + Date.now(),
getSessionLogId: () => "session-scenario",
getModelId: () => "scenario-runner",
getVcsRevisionId: () => undefined,
})
// Step 1: Select intent (like the agent would)
console.log("1. select_active_intent(INT-001) ...")
const selectResult = await middleware.preToolUse("select_active_intent", { intent_id: "INT-001" })
if (selectResult.blocked) {
console.error(" FAIL: select_active_intent blocked:", selectResult.error)
process.exit(1)
}
console.log(" OK intent context loaded\n")
// Step 2: Simulate write_to_file with intent_id and mutation_class
const writeParams = {
path: "src/api/weather.ts",
content: "// Weather API\nconst getWeather = () => ({ temp: 72 })\nexport { getWeather }\n",
intent_id: "INT-001",
mutation_class: "INTENT_EVOLUTION" as const,
}
console.log("2. write_to_file (pre-hook check) ...")
const preWrite = await middleware.preToolUse("write_to_file", writeParams)
if (preWrite.blocked) {
console.error(" FAIL: write_to_file blocked:", preWrite.error)
process.exit(1)
}
console.log(" OK within scope\n")
// Step 3: Post-hook appends to agent_trace.jsonl (like after real write)
console.log("3. postToolUse (append to agent_trace.jsonl) ...")
await middleware.postToolUse("write_to_file", writeParams, {})
console.log(" OK trace entry appended\n")
// Step 4: Read and display the last trace entry
const raw = await fs.readFile(tracePath, "utf-8")
const lines = raw.trim().split("\n").filter(Boolean)
const lastLine = lines[lines.length - 1]
if (!lastLine) {
console.error(" FAIL: no lines in agent_trace.jsonl")
process.exit(1)
}
const entry = JSON.parse(lastLine)
const expectedHash = contentHash(writeParams.content)
console.log("4. Last trace entry (from .orchestration/agent_trace.jsonl):\n")
console.log(JSON.stringify(entry, null, 2))
console.log("\n--- Checks ---")
console.log(
" intent (specification):",
entry.files[0].conversations[0].related?.find((r: { type: string }) => r.type === "specification")?.value ??
"missing",
)
console.log(
" REQ-ID (request): ",
entry.files[0].conversations[0].related?.find((r: { type: string }) => r.type === "request")?.value ??
"missing",
)
console.log(
" content_hash in ranges: ",
entry.files[0].conversations[0].ranges[0]?.content_hash?.startsWith("sha256:") ? "yes" : "no",
)
console.log(
" expected hash match: ",
entry.files[0].conversations[0].ranges[0]?.content_hash === expectedHash ? "yes" : "no",
)
console.log("\n=== Scenario done. ===\n")
}
main().catch((err) => {
console.error(err)
process.exit(1)
})