mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-17 09:31:39 +00:00
test(dsh): diagnose activation probe assertion failures safely
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
parent
8455f4e06e
commit
03cda4fa62
4 changed files with 116 additions and 37 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import { safeAvailabilityCheckpoint } from './smoke-native-host.mjs';
|
||||
import { safeAvailabilityCheckpoint, safeToggleCheckpoint, safeFixtureLocation } from './smoke-native-host.mjs';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { mkdir, readFile, realpath, writeFile, access } from 'node:fs/promises';
|
||||
|
|
@ -30,6 +30,8 @@ let selectedProbe;
|
|||
let observerStatusFile;
|
||||
let dependencyStatusFile;
|
||||
let availabilityCheckpoint;
|
||||
let toggleCheckpoint;
|
||||
let fixtureLocation;
|
||||
let failureCode = 'PROBE_FAILED';
|
||||
function fail(code, message) { failureCode = code; throw new Error(message); }
|
||||
async function main() {
|
||||
|
|
@ -181,6 +183,10 @@ try {
|
|||
if (report) {
|
||||
if (report.passed !== true) {
|
||||
if (values.probe === 'skill-availability') availabilityCheckpoint = safeAvailabilityCheckpoint(report.checkpoint);
|
||||
if (['skill-toggle', 'skill-toggle-watch-off'].includes(values.probe)) {
|
||||
toggleCheckpoint = safeToggleCheckpoint(report.checkpoint);
|
||||
fixtureLocation = safeFixtureLocation(report.location);
|
||||
}
|
||||
stage = 'host-assertion';
|
||||
fail('HOST_ASSERTION_FAILED', 'Host probe failed; inspect its result file.');
|
||||
}
|
||||
|
|
@ -208,6 +214,8 @@ try {
|
|||
main().catch(async () => {
|
||||
const diagnostics = {};
|
||||
if (availabilityCheckpoint) diagnostics.availability = availabilityCheckpoint;
|
||||
if (toggleCheckpoint) diagnostics.toggle = toggleCheckpoint;
|
||||
if (fixtureLocation) diagnostics.location = fixtureLocation;
|
||||
for (const [key, file] of [['observer', observerStatusFile], ['dependencies', dependencyStatusFile]]) {
|
||||
if (!file) continue;
|
||||
try { diagnostics[key] = JSON.parse(await readFile(file, 'utf8')); } catch { /* Missing marker is itself diagnostic. */ }
|
||||
|
|
|
|||
|
|
@ -128,6 +128,19 @@ export function safeAvailabilityCheckpoint(value) {
|
|||
return { scenario: value.scenario, checkpoint: value.checkpoint, assertion: value.assertion };
|
||||
}
|
||||
|
||||
export function safeToggleCheckpoint(value) {
|
||||
if (!value || typeof value !== 'object') return undefined;
|
||||
const scenarios = ['setup', 'install', 'initial-discovery', 'watch-off-disable', 'watch-off-diagnosis', 'watch-off-uninstall', 'cancel', 'disable', 'disabled-files', 'disabled-use', 'enable', 'disable-again', 'disabled-upgrade', 'disabled-uninstall', 'same-name-install', 'same-name-disable', 'same-name-fallback', 'same-name-diagnosis', 'cleanup'];
|
||||
const assertions = ['none', 'invoke', 'operation', 'discovery', 'equal', 'notEqual', 'deepEqual', 'rejects', 'truthy'];
|
||||
if (!scenarios.includes(value.scenario) || !assertions.includes(value.assertion)) return undefined;
|
||||
return { scenario: value.scenario, assertion: value.assertion };
|
||||
}
|
||||
|
||||
export function safeFixtureLocation(value) {
|
||||
if (value?.fixture !== 'skill-toggle-fixture.mjs' || !Number.isInteger(value.line) || value.line < 1 || value.line > 10000) return undefined;
|
||||
return { fixture: value.fixture, line: value.line };
|
||||
}
|
||||
|
||||
export function safeProbeFailure(log, probe) {
|
||||
const stages = ['setup', 'validate-root', 'prepare-fixture', 'fixture-ports', 'registry-ready', 'host-result', 'host-assertion'];
|
||||
const codes = ['PROBE_FAILED', 'REGISTRY_EXITED', 'REGISTRY_TIMEOUT', 'CHILD_EXITED', 'HOST_ASSERTION_FAILED', 'HOST_RESULT_TIMEOUT'];
|
||||
|
|
@ -147,6 +160,12 @@ export function safeProbeFailure(log, probe) {
|
|||
}
|
||||
const availability = safeAvailabilityCheckpoint(result.diagnostics.availability);
|
||||
if (probe === 'skill-availability' && availability) safe.availability = availability;
|
||||
if (['skill-toggle', 'skill-toggle-watch-off'].includes(probe)) {
|
||||
const toggle = safeToggleCheckpoint(result.diagnostics.toggle);
|
||||
const location = safeFixtureLocation(result.diagnostics.location);
|
||||
if (toggle) safe.toggle = toggle;
|
||||
if (location) safe.location = location;
|
||||
}
|
||||
safe.dependencies = {};
|
||||
for (const name of ['loader', 'pluginPackages', 'clientModules', 'skillhub']) {
|
||||
const present = result.diagnostics.dependencies?.[name];
|
||||
|
|
|
|||
|
|
@ -17,9 +17,16 @@ export function apply(ctx, config) {
|
|||
assert.equal(process.env.DSH_HOME, join(root, 'home'));
|
||||
assert.equal(process.env.DSH_AGENTS_HOME, join(root, 'agents'));
|
||||
assert.equal(process.env.HOME, join(root, 'native-install-home'));
|
||||
const checkpoint = { scenario: 'setup', assertion: 'none' };
|
||||
const abort = new AbortController();
|
||||
const timer = setTimeout(() => {
|
||||
run(ctx, root, abort.signal, config.watchOff === true).then(result => ({ passed: true, ...result }), error => ({ passed: false, error: error.message }))
|
||||
run(ctx, root, abort.signal, config.watchOff === true, checkpoint).then(result => ({ passed: true, ...result }), error => {
|
||||
// Only this copied fixture's bounded line number may leave the private result.
|
||||
const frame = String(error.stack ?? '').split('\n').filter(line => line.includes(fileURLToPath(import.meta.url)) || line.includes(import.meta.url)).at(-1);
|
||||
const line = Number(/:(\d+):\d+\)?$/.exec(frame ?? '')?.[1]);
|
||||
const location = Number.isInteger(line) && line >= 1 && line <= 10000 ? { fixture: 'skill-toggle-fixture.mjs', line } : undefined;
|
||||
return { passed: false, error: error.message, checkpoint, location };
|
||||
})
|
||||
.then(result => writeFile(join(root, 'skill-toggle-result.json'), JSON.stringify(result, null, 2))).catch(() => {});
|
||||
}, 1000);
|
||||
return () => { clearTimeout(timer); abort.abort(); };
|
||||
|
|
@ -38,7 +45,12 @@ async function hashes(directory) {
|
|||
return result;
|
||||
}
|
||||
|
||||
async function run(ctx, root, signal, watchOff) {
|
||||
async function run(ctx, root, signal, watchOff, checkpoint) {
|
||||
const checked = Object.assign((...args) => { checkpoint.assertion = 'truthy'; return assert(...args); },
|
||||
Object.fromEntries(['equal', 'notEqual', 'deepEqual', 'rejects'].map(name => [name, (...args) => {
|
||||
checkpoint.assertion = name; return assert[name](...args);
|
||||
}])));
|
||||
|
||||
const project = join(root, 'toggle-project-' + randomUUID());
|
||||
await mkdir(join(project, '.git'), { recursive: true });
|
||||
const workspace = await ctx.workspaceRegistry.create(project);
|
||||
|
|
@ -47,32 +59,35 @@ async function run(ctx, root, signal, watchOff) {
|
|||
if ('error' in resolved) throw resolved.error;
|
||||
let agent = resolved.agent;
|
||||
const options = { scope: agent, cwd: agent.session.header.cwd, signal };
|
||||
const invoke = (method, extra = {}) => ctx.typertGateway.invoke({
|
||||
const invoke = (method, extra = {}) => { checkpoint.assertion = 'invoke'; return ctx.typertGateway.invoke({
|
||||
peer: ctx.get('connection')?.browserPeer ?? ctx.get('connection')?.operator,
|
||||
namespace: 'skillhub', method, args: { ...(method === 'getSkillDetail' ? {} : { agentId: agent.session.id }), ...extra }, signal,
|
||||
});
|
||||
}); };
|
||||
const source = ctx.skillhub.getContext();
|
||||
const sourceRevision = source.sourceRevision;
|
||||
const detail = await invoke('getSkillDetail', { sourceRevision, namespace: process.env.DSH_FIXTURE_NAMESPACE ?? 'demo', slug: 'document-helper', version: '1.2.0' });
|
||||
const targets = await invoke('listInstallTargets', { detailToken: detail.detailToken });
|
||||
const target = targets.targets.find(item => item.scope === 'project'); assert(target);
|
||||
const target = targets.targets.find(item => item.scope === 'project'); checked(target);
|
||||
const operations = [];
|
||||
async function complete(receipt) {
|
||||
assert(receipt); const deadline = Date.now() + 20000;
|
||||
checkpoint.assertion = 'operation';
|
||||
checked(receipt); const deadline = Date.now() + 20000;
|
||||
while (Date.now() < deadline) {
|
||||
const operation = await invoke('getOperation', { operationId: receipt.operationId, hostInstanceId: receipt.context.hostInstanceId });
|
||||
if (operation.phase === 'completed') { operations.push({ kind: operation.kind, files: operation.files, discovery: operation.discovery }); return operation; }
|
||||
assert(!['failed', 'recovery-required', 'unknown'].includes(operation.phase), JSON.stringify(operation));
|
||||
checked(!['failed', 'recovery-required', 'unknown'].includes(operation.phase), JSON.stringify(operation));
|
||||
await delay(50, undefined, { signal });
|
||||
}
|
||||
throw new Error('Operation timed out');
|
||||
}
|
||||
checkpoint.scenario = 'install';
|
||||
await complete(await invoke('startInstall', { input: { detailToken: detail.detailToken, targetToken: target.targetToken,
|
||||
hostInstanceId: targets.hostInstanceId, requestKey: randomUUID(), mode: 'new', initiatedFrom: 'plaza' } }));
|
||||
const directory = join(project, '.dsh/skills/document-helper');
|
||||
const before = await hashes(directory);
|
||||
const list = async () => (await invoke('listInstalled', { sourceRevision })).items.find(item => item.scope === 'project' && item.slug === 'document-helper');
|
||||
async function discovered(present) {
|
||||
checkpoint.assertion = 'discovery';
|
||||
const deadline = Date.now() + 15000;
|
||||
while (Date.now() < deadline) {
|
||||
const snapshot = await ctx.skills.snapshot(options);
|
||||
|
|
@ -82,63 +97,78 @@ async function run(ctx, root, signal, watchOff) {
|
|||
}
|
||||
throw new Error(`Full Host discovery did not become ${present}`);
|
||||
}
|
||||
const loaded = await discovered(true); assert.equal(loaded.resourceBase.path, directory);
|
||||
checkpoint.scenario = 'initial-discovery';
|
||||
const loaded = await discovered(true); checked.equal(loaded.resourceBase.path, directory);
|
||||
async function mutate(action, item, version = null) {
|
||||
const prepared = await invoke('beginConversationManagement', { sourceRevision, itemKey: item.itemKey, action, version, requestKey: randomUUID() });
|
||||
assert.equal(prepared.status, 'awaiting-confirmation', JSON.stringify(prepared));
|
||||
checked.equal(prepared.status, 'awaiting-confirmation', JSON.stringify(prepared));
|
||||
const receipt = await invoke('confirmConversationManagement', { confirmationRef: prepared.confirmationRef, confirmed: true });
|
||||
assert.equal(receipt.status, 'accepted'); return complete(receipt.operation);
|
||||
checked.equal(receipt.status, 'accepted'); return complete(receipt.operation);
|
||||
}
|
||||
const original = await list();
|
||||
if (watchOff) {
|
||||
checkpoint.scenario = 'watch-off-disable';
|
||||
await mutate('disable', original);
|
||||
checkpoint.scenario = 'watch-off-diagnosis';
|
||||
const selected = await list();
|
||||
const diagnosis = await invoke('checkAvailability', { sourceRevision, itemKey: selected.itemKey });
|
||||
assert.equal(diagnosis.installationState, 'disabled');
|
||||
assert.equal(diagnosis.integrity, 'verified');
|
||||
assert.equal((await invoke('prepareSkillUse', { sourceRevision, itemKey: selected.itemKey })).reason, 'installation-disabled');
|
||||
assert.equal(operations.at(-1).discovery, 'unconfirmed');
|
||||
checked.equal(diagnosis.installationState, 'disabled');
|
||||
checked.equal(diagnosis.integrity, 'verified');
|
||||
checked.equal((await invoke('prepareSkillUse', { sourceRevision, itemKey: selected.itemKey })).reason, 'installation-disabled');
|
||||
checked.equal(operations.at(-1).discovery, 'unconfirmed');
|
||||
const snapshot = await ctx.skills.snapshot(options);
|
||||
const cached = await ctx.skills.get('document-helper', options);
|
||||
checkpoint.scenario = 'watch-off-uninstall';
|
||||
await mutate('uninstall', selected);
|
||||
return { operations, watchOff: true, diagnosis: { installationState: diagnosis.installationState,
|
||||
discovery: diagnosis.discovery, reason: diagnosis.reason },
|
||||
snapshotComplete: snapshot.complete, snapshotContainsName: snapshot.skills.some(item => item.name === 'document-helper'),
|
||||
loadedAfterDisable: !!cached, noInstantRevocationClaim: true };
|
||||
}
|
||||
checkpoint.scenario = 'cancel';
|
||||
const cancelled = await invoke('beginConversationManagement', { sourceRevision, itemKey: original.itemKey, action: 'disable', version: null, requestKey: randomUUID() });
|
||||
assert.equal((await invoke('confirmConversationManagement', { confirmationRef: cancelled.confirmationRef, confirmed: false })).status, 'cancelled');
|
||||
assert.deepEqual(await hashes(directory), before);
|
||||
checked.equal((await invoke('confirmConversationManagement', { confirmationRef: cancelled.confirmationRef, confirmed: false })).status, 'cancelled');
|
||||
checked.deepEqual(await hashes(directory), before);
|
||||
checkpoint.scenario = 'disable';
|
||||
await mutate('disable', original);
|
||||
await discovered(false);
|
||||
let item = await list(); assert.equal(item.installationState, 'disabled'); assert.equal(item.state, 'present');
|
||||
assert.equal(item.locationKey, original.locationKey); assert.notEqual(item.itemKey, original.itemKey);
|
||||
let item = await list(); checked.equal(item.installationState, 'disabled'); checked.equal(item.state, 'present');
|
||||
checked.equal(item.locationKey, original.locationKey); checked.notEqual(item.itemKey, original.itemKey);
|
||||
checkpoint.scenario = 'disabled-files';
|
||||
const disabled = await hashes(directory);
|
||||
assert.equal(disabled['SKILL.md.disabled'], before['SKILL.md']); assert(!('SKILL.md' in disabled));
|
||||
checked.equal(disabled['SKILL.md.disabled'], before['SKILL.md']); checked(!('SKILL.md' in disabled));
|
||||
const canonical = { ...disabled, 'SKILL.md': disabled['SKILL.md.disabled'] }; delete canonical['SKILL.md.disabled'];
|
||||
assert.deepEqual(canonical, before);
|
||||
assert.equal(await readFile(join(loaded.resourceBase.path, 'references/note.md'), 'utf8'), 'Native installation resource');
|
||||
await assert.rejects(invoke('prepareSkillUse', { sourceRevision, itemKey: original.itemKey }));
|
||||
assert.equal((await invoke('prepareSkillUse', { sourceRevision, itemKey: item.itemKey })).reason, 'installation-disabled');
|
||||
await mutate('enable', item); await discovered(true); assert.deepEqual(await hashes(directory), before);
|
||||
checked.deepEqual(canonical, before);
|
||||
checked.equal(await readFile(join(loaded.resourceBase.path, 'references/note.md'), 'utf8'), 'Native installation resource');
|
||||
checkpoint.scenario = 'disabled-use';
|
||||
await checked.rejects(invoke('prepareSkillUse', { sourceRevision, itemKey: original.itemKey }));
|
||||
checked.equal((await invoke('prepareSkillUse', { sourceRevision, itemKey: item.itemKey })).reason, 'installation-disabled');
|
||||
checkpoint.scenario = 'enable';
|
||||
await mutate('enable', item); await discovered(true); checked.deepEqual(await hashes(directory), before);
|
||||
checkpoint.scenario = 'disable-again';
|
||||
item = await list(); await mutate('disable', item); await discovered(false);
|
||||
checkpoint.scenario = 'disabled-upgrade';
|
||||
item = await list(); await mutate('upgrade', item, '1.3.0');
|
||||
item = await list(); assert.equal(item.installationState, 'disabled'); assert.equal(item.version, '1.3.0');
|
||||
assert.equal(await readFile(join(directory, 'references/note.md'), 'utf8'), 'Native upgraded resource 1.3.0');
|
||||
assert(!('SKILL.md' in await hashes(directory))); await discovered(false);
|
||||
await mutate('uninstall', item); assert.equal(await list(), undefined); await discovered(false);
|
||||
item = await list(); checked.equal(item.installationState, 'disabled'); checked.equal(item.version, '1.3.0');
|
||||
checked.equal(await readFile(join(directory, 'references/note.md'), 'utf8'), 'Native upgraded resource 1.3.0');
|
||||
checked(!('SKILL.md' in await hashes(directory))); await discovered(false);
|
||||
checkpoint.scenario = 'disabled-uninstall';
|
||||
await mutate('uninstall', item); checked.equal(await list(), undefined); await discovered(false);
|
||||
// Independent same-name user/project installations: only the project entry is disabled.
|
||||
const installScope = async scope => {
|
||||
const targets = await invoke('listInstallTargets', { detailToken: detail.detailToken });
|
||||
const selected = targets.targets.find(item => item.scope === scope); assert(selected);
|
||||
const selected = targets.targets.find(item => item.scope === scope); checked(selected);
|
||||
await complete(await invoke('startInstall', { input: { detailToken: detail.detailToken, targetToken: selected.targetToken,
|
||||
hostInstanceId: targets.hostInstanceId, requestKey: randomUUID(), mode: 'new', initiatedFrom: 'plaza' } }));
|
||||
};
|
||||
checkpoint.scenario = 'same-name-install';
|
||||
await installScope('user'); await installScope('project');
|
||||
await discovered(true);
|
||||
const userDirectory = join(root, 'home/skills/document-helper');
|
||||
const userBefore = await hashes(userDirectory);
|
||||
checkpoint.scenario = 'same-name-disable';
|
||||
await mutate('disable', await list());
|
||||
checkpoint.scenario = 'same-name-fallback';
|
||||
const deadline = Date.now() + 15000;
|
||||
let fallback;
|
||||
do {
|
||||
|
|
@ -150,21 +180,23 @@ async function run(ctx, root, signal, watchOff) {
|
|||
const originalSelected = await list();
|
||||
const originalDiagnosis = await invoke('checkAvailability', { sourceRevision, itemKey: originalSelected.itemKey });
|
||||
if (!originalSessionFallback) {
|
||||
assert.equal(originalDiagnosis.installationState, 'disabled');
|
||||
assert.equal(originalDiagnosis.discovery, 'unknown');
|
||||
checked.equal(originalDiagnosis.installationState, 'disabled');
|
||||
checked.equal(originalDiagnosis.discovery, 'unknown');
|
||||
const fresh = await ctx.sessionController.create({ workspaceId: workspace.id, agentPreset: 'standard' });
|
||||
const current = await ctx.sessionController.resolveAgent(fresh.sessionId);
|
||||
if ('error' in current) throw current.error;
|
||||
agent = current.agent; options.scope = agent;
|
||||
await ctx.skills.snapshot(options); fallback = await ctx.skills.get('document-helper', options);
|
||||
}
|
||||
assert.equal(fallback?.resourceBase?.path, userDirectory);
|
||||
checked.equal(fallback?.resourceBase?.path, userDirectory);
|
||||
checkpoint.scenario = 'same-name-diagnosis';
|
||||
const selected = await list();
|
||||
const diagnosis = await invoke('checkAvailability', { sourceRevision, itemKey: selected.itemKey });
|
||||
assert.equal(diagnosis.installationState, 'disabled'); assert.equal(diagnosis.discovery, 'shadowed');
|
||||
assert.equal(diagnosis.reason, 'same-name-winner'); assert.equal(diagnosis.name, 'document-helper');
|
||||
assert.deepEqual(await hashes(userDirectory), userBefore);
|
||||
assert.equal((await invoke('prepareSkillUse', { sourceRevision, itemKey: selected.itemKey })).reason, 'installation-disabled');
|
||||
checked.equal(diagnosis.installationState, 'disabled'); checked.equal(diagnosis.discovery, 'shadowed');
|
||||
checked.equal(diagnosis.reason, 'same-name-winner'); checked.equal(diagnosis.name, 'document-helper');
|
||||
checked.deepEqual(await hashes(userDirectory), userBefore);
|
||||
checked.equal((await invoke('prepareSkillUse', { sourceRevision, itemKey: selected.itemKey })).reason, 'installation-disabled');
|
||||
checkpoint.scenario = 'cleanup';
|
||||
await mutate('uninstall', selected);
|
||||
const userItem = (await invoke('listInstalled', { sourceRevision })).items.find(item => item.scope === 'user' && item.slug === 'document-helper');
|
||||
await mutate('uninstall', userItem);
|
||||
|
|
|
|||
|
|
@ -41,3 +41,23 @@ test('availability diagnostics retain only fixed checkpoint labels, never raw fa
|
|||
}
|
||||
expect(safeProbeFailure(JSON.stringify({ ...base, probe: 'plugin-lifecycle' }), 'plugin-lifecycle').availability).toBeUndefined();
|
||||
});
|
||||
|
||||
test('both toggle probes expose only bounded fixture locations and fixed labels', () => {
|
||||
for (const probe of ['skill-toggle', 'skill-toggle-watch-off']) {
|
||||
const base = { passed: false, probe, stage: 'host-assertion', failureCode: 'HOST_ASSERTION_FAILED',
|
||||
diagnostics: { toggle: { scenario: 'disabled-upgrade', assertion: 'equal', error: 'private-secret' },
|
||||
location: { fixture: 'skill-toggle-fixture.mjs', line: 152, stack: '/private/secret' } } };
|
||||
const safe = safeProbeFailure(JSON.stringify(base), probe);
|
||||
expect(safe.toggle).toEqual({ scenario: 'disabled-upgrade', assertion: 'equal' });
|
||||
expect(safe.location).toEqual({ fixture: 'skill-toggle-fixture.mjs', line: 152 });
|
||||
expect(JSON.stringify(safe)).not.toMatch(/private|secret|stack/);
|
||||
for (const line of [0, -1, 10001, 1.5, '152', null]) {
|
||||
expect(safeProbeFailure(JSON.stringify({ ...base, diagnostics: { location: { fixture: 'skill-toggle-fixture.mjs', line } } }), probe).location).toBeUndefined();
|
||||
}
|
||||
expect(safeProbeFailure(JSON.stringify({ ...base, diagnostics: { location: { fixture: '/private/skill-toggle-fixture.mjs', line: 3 } } }), probe).location).toBeUndefined();
|
||||
for (const key of ['scenario', 'assertion']) {
|
||||
expect(safeProbeFailure(JSON.stringify({ ...base, diagnostics: { toggle: { ...base.diagnostics.toggle, [key]: 'private-secret' } } }), probe).toggle).toBeUndefined();
|
||||
}
|
||||
expect(safeProbeFailure(JSON.stringify({ ...base, probe: 'runtime-baseline' }), 'runtime-baseline').location).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue