test(dsh): capture sanitized operation terminal diagnostics

Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
XiaoSeS 2026-10-29 18:56:47 +08:00
parent 03cda4fa62
commit 8b54c2c07c
5 changed files with 75 additions and 4 deletions

View file

@ -1,7 +1,7 @@
import { safeAvailabilityCheckpoint, safeToggleCheckpoint, safeFixtureLocation } from './smoke-native-host.mjs';
import { safeAvailabilityCheckpoint, safeToggleCheckpoint, safeFixtureLocation, safeOperationObservation, knownHostFailureCodes } from './smoke-native-host.mjs';
import { spawn } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { mkdir, readFile, realpath, writeFile, access } from 'node:fs/promises';
import { mkdir, readFile, realpath, writeFile, access, open } from 'node:fs/promises';
import { openSync, closeSync } from 'node:fs';
import { createServer } from 'node:net';
import { tmpdir } from 'node:os';
@ -32,6 +32,8 @@ let dependencyStatusFile;
let availabilityCheckpoint;
let toggleCheckpoint;
let fixtureLocation;
let operationObservation;
let hostLogFile;
let failureCode = 'PROBE_FAILED';
function fail(code, message) { failureCode = code; throw new Error(message); }
async function main() {
@ -81,6 +83,7 @@ await mkdir(run, { mode: 0o700 });
const resultName = `probe-${id}-result.json`;
const resultFile = join(root, resultName);
const observer = join(run, 'observer.mjs');
hostLogFile = join(run, 'host.log');
observerStatusFile = join(run, 'observer-status.json');
dependencyStatusFile = join(run, 'dependency-status.json');
const runtimeProbe = values.probe === 'runtime-baseline' || values.probe === 'runtime-unsupported';
@ -182,6 +185,7 @@ try {
catch (error) { if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error; }
if (report) {
if (report.passed !== true) {
if (['skill-availability', 'skill-toggle', 'skill-toggle-watch-off'].includes(values.probe)) operationObservation = safeOperationObservation(report.operation);
if (values.probe === 'skill-availability') availabilityCheckpoint = safeAvailabilityCheckpoint(report.checkpoint);
if (['skill-toggle', 'skill-toggle-watch-off'].includes(values.probe)) {
toggleCheckpoint = safeToggleCheckpoint(report.checkpoint);
@ -216,6 +220,18 @@ main().catch(async () => {
if (availabilityCheckpoint) diagnostics.availability = availabilityCheckpoint;
if (toggleCheckpoint) diagnostics.toggle = toggleCheckpoint;
if (fixtureLocation) diagnostics.location = fixtureLocation;
if (operationObservation) diagnostics.operation = operationObservation;
if (hostLogFile && ['skill-availability', 'skill-toggle', 'skill-toggle-watch-off'].includes(selectedProbe)) {
let file;
try {
file = await open(hostLogFile, 'r');
const { size } = await file.stat();
const bytes = Buffer.alloc(Math.min(size, 1024 * 1024));
const { bytesRead } = await file.read(bytes, 0, bytes.length, Math.max(0, size - bytes.length));
diagnostics.observedHostCodes = knownHostFailureCodes(bytes.subarray(0, bytesRead).toString('utf8'));
} catch { /* Diagnostics must not replace the original failure. */ }
finally { await file?.close().catch(() => {}); }
}
for (const [key, file] of [['observer', observerStatusFile], ['dependencies', dependencyStatusFile]]) {
if (!file) continue;
try { diagnostics[key] = JSON.parse(await readFile(file, 'utf8')); } catch { /* Missing marker is itself diagnostic. */ }

View file

@ -141,6 +141,28 @@ export function safeFixtureLocation(value) {
return { fixture: value.fixture, line: value.line };
}
export function safeOperationObservation(value) {
if (!value || typeof value !== 'object') return undefined;
const fields = {
phase: ['accepted', 'staging', 'revalidating', 'committing', 'recovering', 'completed', 'failed', 'recovery-required', 'unknown'],
files: ['pending', 'installed', 'already-installed', 'upgraded', 'removed', 'stale-record-removed', 'enabled', 'disabled', 'failed', 'unknown'],
discovery: ['unconfirmed', 'confirmed', 'failed'], recovery: ['not-needed', 'restored', 'incomplete', 'unknown'],
};
const safe = {};
for (const [name, allowed] of Object.entries(fields)) if (allowed.includes(value[name])) safe[name] = value[name];
if (typeof value.deadlineReached === 'boolean') safe.deadlineReached = value.deadlineReached;
return Object.keys(safe).length ? safe : undefined;
}
const diagnosticCodes = ['EACCES', 'EPERM', 'ENOENT', 'EEXIST', 'ENOSPC', 'EBUSY', 'EXDEV', 'ENOTEMPTY', 'EIO', 'ELOCKED', 'ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN', 'AUTHORIZATION', 'NETWORK', 'FILESYSTEM', 'INVALID_INPUT', 'VALIDATION', 'UNCLASSIFIED'];
export function safeHostFailureCodes(value) {
return Array.isArray(value) ? diagnosticCodes.filter(code => value.includes(code)) : [];
}
export function knownHostFailureCodes(log) {
// Observed codes are diagnostic hints, not attribution of the assertion's cause.
return diagnosticCodes.filter(code => new RegExp('\\b' + code + '\\b').test(log));
}
export function safeProbeFailure(log, probe) {
const stages = ['setup', 'validate-root', 'prepare-fixture', 'fixture-ports', 'registry-ready', 'host-result', 'host-assertion'];
const codes = ['PROBE_FAILED', 'REGISTRY_EXITED', 'REGISTRY_TIMEOUT', 'CHILD_EXITED', 'HOST_ASSERTION_FAILED', 'HOST_RESULT_TIMEOUT'];
@ -166,6 +188,12 @@ export function safeProbeFailure(log, probe) {
if (toggle) safe.toggle = toggle;
if (location) safe.location = location;
}
if (['skill-availability', 'skill-toggle', 'skill-toggle-watch-off'].includes(probe)) {
const operation = safeOperationObservation(result.diagnostics.operation);
if (operation) safe.operation = operation;
const codes = safeHostFailureCodes(result.diagnostics.observedHostCodes);
if (codes.length) safe.observedHostCodes = codes;
}
safe.dependencies = {};
for (const name of ['loader', 'pluginPackages', 'clientModules', 'skillhub']) {
const present = result.diagnostics.dependencies?.[name];

View file

@ -15,7 +15,7 @@ export function apply(ctx, config) {
const checkpoint = { scenario: 'setup', checkpoint: 'setup', assertion: 'none' };
const abort = new AbortController();
const timer = setTimeout(() => run(ctx, root, AbortSignal.any([abort.signal, AbortSignal.timeout(70000)]), checkpoint)
.catch(error => writeFile(join(root, 'skill-availability-result.json'), JSON.stringify({ passed: false, error: error.message, checkpoint }))), 1000);
.catch(error => writeFile(join(root, 'skill-availability-result.json'), JSON.stringify({ passed: false, error: error.message, checkpoint, operation: checkpoint.operation }))), 1000);
return () => { clearTimeout(timer); abort.abort(); };
}
@ -94,6 +94,7 @@ async function run(ctx, root, signal, checkpoint) {
return result.value.diagnosis;
}
async function install(scope) {
checkpoint.operation = { deadlineReached: false };
mark('install');
const result = await execute('astron_skillhub_install', { namespace, slug: 'document-helper', scope });
mark('assertion', 'isError');
@ -105,9 +106,12 @@ async function run(ctx, root, signal, checkpoint) {
mark('install-poll');
while (Date.now() < deadline) {
operation = await ctx.skillhub.getInstallationOperation(created.sessionId, result.value.operationId);
checkpoint.operation = { phase: operation?.phase, files: operation?.files, recovery: operation?.recovery,
discovery: operation?.discovery, deadlineReached: Date.now() >= deadline };
if (['completed', 'failed', 'unknown', 'recovery-required'].includes(operation?.phase)) break;
await delay(100, undefined, { signal });
}
checkpoint.operation.deadlineReached = Date.now() >= deadline;
mark('assertion', 'completed');
assert.equal(operation?.phase, 'completed');
}

View file

@ -25,7 +25,7 @@ export function apply(ctx, config) {
const frame = String(error.stack ?? '').split('\n').filter(line => line.includes(fileURLToPath(import.meta.url)) || line.includes(import.meta.url)).at(-1);
const line = Number(/:(\d+):\d+\)?$/.exec(frame ?? '')?.[1]);
const location = Number.isInteger(line) && line >= 1 && line <= 10000 ? { fixture: 'skill-toggle-fixture.mjs', line } : undefined;
return { passed: false, error: error.message, checkpoint, location };
return { passed: false, error: error.message, checkpoint, location, operation: checkpoint.operation };
})
.then(result => writeFile(join(root, 'skill-toggle-result.json'), JSON.stringify(result, null, 2))).catch(() => {});
}, 1000);
@ -70,14 +70,18 @@ async function run(ctx, root, signal, watchOff, checkpoint) {
const target = targets.targets.find(item => item.scope === 'project'); checked(target);
const operations = [];
async function complete(receipt) {
checkpoint.operation = { deadlineReached: false };
checkpoint.assertion = 'operation';
checked(receipt); const deadline = Date.now() + 20000;
while (Date.now() < deadline) {
const operation = await invoke('getOperation', { operationId: receipt.operationId, hostInstanceId: receipt.context.hostInstanceId });
checkpoint.operation = { phase: operation?.phase, files: operation?.files, recovery: operation?.recovery,
discovery: operation?.discovery, deadlineReached: Date.now() >= deadline };
if (operation.phase === 'completed') { operations.push({ kind: operation.kind, files: operation.files, discovery: operation.discovery }); return operation; }
checked(!['failed', 'recovery-required', 'unknown'].includes(operation.phase), JSON.stringify(operation));
await delay(50, undefined, { signal });
}
checkpoint.operation.deadlineReached = true;
throw new Error('Operation timed out');
}
checkpoint.scenario = 'install';

View file

@ -61,3 +61,22 @@ test('both toggle probes expose only bounded fixture locations and fixed labels'
expect(safeProbeFailure(JSON.stringify({ ...base, probe: 'runtime-baseline' }), 'runtime-baseline').location).toBeUndefined();
}
});
test('operation observations and Host codes whitelist every field without revealing operation bodies', async () => {
// @ts-expect-error Development-only Node script has no declarations.
const { knownHostFailureCodes } = await import('../scripts/smoke-native-host.mjs');
expect(knownHostFailureCodes('private /path EPERM code=EBUSY arbitrary-secret NOT_EIO EPERM')).toEqual(['EPERM', 'EBUSY']);
for (const probe of ['skill-availability', 'skill-toggle', 'skill-toggle-watch-off']) {
const base = { passed: false, probe, stage: 'host-assertion', failureCode: 'HOST_ASSERTION_FAILED', diagnostics: {
operation: { phase: 'failed', files: 'failed', recovery: 'restored', discovery: 'unconfirmed', deadlineReached: false,
operationId: 'secret', targetLabel: '/private', reason: 'private-secret', code: 'SECRET', error: 'raw-secret' },
observedHostCodes: ['EPERM', '/private', 'EBUSY', 'EPERM', 'SECRET'],
} };
const safe = safeProbeFailure(JSON.stringify(base), probe);
expect(safe.operation).toEqual({ phase: 'failed', files: 'failed', recovery: 'restored', discovery: 'unconfirmed', deadlineReached: false });
expect(safe.observedHostCodes).toEqual(['EPERM', 'EBUSY']);
expect(JSON.stringify(safe)).not.toMatch(/private|secret|SECRET|operationId|targetLabel|reason/);
expect(safeProbeFailure(JSON.stringify({ ...base, diagnostics: { operation: { phase: '/secret', files: '/secret', discovery: '/secret', recovery: '/secret', deadlineReached: 'secret' }, observedHostCodes: 'EPERM' } }), probe).operation).toBeUndefined();
expect(safeProbeFailure(JSON.stringify({ ...base, probe: 'runtime-baseline' }), 'runtime-baseline').operation).toBeUndefined();
}
});