diff --git a/cli/src/integrations/operation-store.ts b/cli/src/integrations/operation-store.ts index 4c2a0fc9b..4fecab694 100644 --- a/cli/src/integrations/operation-store.ts +++ b/cli/src/integrations/operation-store.ts @@ -1,9 +1,10 @@ +import { renameAtomicFile } from '../platform/atomic-rename' import { inventoryWithoutState, type Inventory } from '../stores/inventory-store' import { isDeepStrictEqual } from 'node:util' import { toggleJournal, validateToggleJournal, type ToggleJournal } from './toggle-journal' import { createHash, randomUUID } from 'node:crypto' import { constants } from 'node:fs' -import { chmod, link, lstat, mkdir, open, readdir, readFile, realpath, rename, rm, unlink } from 'node:fs/promises' +import { chmod, link, lstat, mkdir, open, readdir, readFile, realpath, rm, unlink } from 'node:fs/promises' import { basename, dirname, isAbsolute, join } from 'node:path' import { canonicalizeExistingPath, pathsOverlap } from '../platform/paths' import { z } from 'zod' @@ -53,21 +54,22 @@ export function operationRequestDigest(parameters: Record { + static async open(root: string, replaceFile: typeof renameAtomicFile = renameAtomicFile): Promise { if (!isAbsolute(root)) throw new Error('Operation state requires a trusted absolute directory') await mkdir(root, { recursive: true, mode: 0o700 }) const info = await lstat(root) if (!info.isDirectory() || info.isSymbolicLink() || (process.getuid && info.uid !== process.getuid())) throw new Error('Unsafe operation state directory') if (process.platform !== 'win32') await chmod(root, 0o700) - const pending = new OperationStore(root, '') + const pending = new OperationStore(root, '', replaceFile) return pending.locked(async () => { const path = join(root, 'host.json') const saved = await readJson(path) const hostInstanceId = saved === undefined ? randomUUID() : z.object({ hostInstanceId: id }).strict().parse(saved).hostInstanceId - if (saved === undefined) await atomicWrite(path, { hostInstanceId }) - const store = new OperationStore(root, hostInstanceId) + if (saved === undefined) await atomicWrite(path, { hostInstanceId }, replaceFile) + const store = new OperationStore(root, hostInstanceId, replaceFile) await store.isolateExtendedRecords() return store }) @@ -483,7 +485,7 @@ export class OperationStore { if (extended) { await this.prepareExtendedDirectory() await this.isolateRecord(value.operationId) - await writeOperationRecord(this.extendedPath(value.operationId), { ...value, schemaVersion: 3 }) + await writeOperationRecord(this.extendedPath(value.operationId), { ...value, schemaVersion: 3 }, this.replaceFile) return } // v2 snapshots contain ownership only, exactly as consumed by published clients. @@ -500,7 +502,7 @@ export class OperationStore { compatible.removal.inventoryCommit.after = inventoryWithoutState(compatible.removal.inventoryCommit.after) } } - await writeOperationRecord(this.path(value.operationId), compatible) + await writeOperationRecord(this.path(value.operationId), compatible, this.replaceFile) } private async locked(run: () => Promise): Promise { @@ -570,14 +572,14 @@ async function readJson(path: string): Promise { } finally { await file.close() } } -async function atomicWrite(path: string, value: unknown): Promise { +async function atomicWrite(path: string, value: unknown, replaceFile: typeof renameAtomicFile): Promise { const temporary = `${path}.${randomUUID()}.tmp` try { const payload = JSON.stringify(value) if (Buffer.byteLength(payload) > 1024 * 1024) throw new OperationStoreError('CORRUPT_RECORD') const file = await open(temporary, 'wx', 0o600) try { await file.writeFile(payload); await file.sync() } finally { await file.close() } - await rename(temporary, path) + await replaceFile(temporary, path) if (process.platform !== 'win32') { const directory = await open(dirname(path), 'r') try { await directory.sync() } finally { await directory.close() } @@ -598,10 +600,10 @@ async function syncJournalDirectory(path: string): Promise { } /** Recovery may advance v1 status, but never rewrite the original serialized evidence. */ -async function writeOperationRecord(path: string, value: OperationRecord): Promise { - if (value.schemaVersion === 3) return atomicWrite(path, value) +async function writeOperationRecord(path: string, value: OperationRecord, replaceFile: typeof renameAtomicFile): Promise { + if (value.schemaVersion === 3) return atomicWrite(path, value, replaceFile) const previous = await readJson(path) as OperationRecord | undefined - if (!previous && value.schemaVersion === 2) return atomicWrite(path, value) + if (!previous && value.schemaVersion === 2) return atomicWrite(path, value, replaceFile) if (!previous || previous.schemaVersion !== value.schemaVersion) throw new Error('Legacy operation evidence missing') const preserved = structuredClone(value) if (preserved.transaction && previous.transaction) { @@ -620,7 +622,7 @@ async function writeOperationRecord(path: string, value: OperationRecord): Promi preserved.removal.inventoryCommit.after = previous.removal.inventoryCommit.after } } - await atomicWrite(path, preserved) + await atomicWrite(path, preserved, replaceFile) } /** A pending install may not exist yet; resolve the nearest existing ancestor. */ diff --git a/cli/src/services/skill-target-lock.ts b/cli/src/services/skill-target-lock.ts index ed1510375..ef14cf585 100644 --- a/cli/src/services/skill-target-lock.ts +++ b/cli/src/services/skill-target-lock.ts @@ -21,7 +21,8 @@ export async function orderSkillTargetLocks(targe } /** Serializes every local lifecycle mutation for one Skill target directory. */ -export async function acquireSkillTargetLock(rootDir: string, slug: string): Promise<() => Promise> { +export async function acquireSkillTargetLock(rootDir: string, slug: string, + acquire: typeof acquireTargetLock = acquireTargetLock): Promise<() => Promise> { const lockPath = await skillTargetLockPath(rootDir, slug) // proper-lockfile's stale deletion is not serialized. Gate only the acquisition attempt so two // recoverers cannot remove and replace the same target lock concurrently. @@ -36,7 +37,7 @@ export async function acquireSkillTargetLock(rootDir: string, slug: string): Pro let releaseTarget: () => Promise try { - releaseTarget = await acquireTargetLock(lockPath) + releaseTarget = await acquire(lockPath) } catch (operationError) { try { await releaseAcquisitionGate() @@ -61,14 +62,27 @@ export async function acquireSkillTargetLock(rootDir: string, slug: string): Pro return releaseTarget } -function acquireTargetLock(lockPath: string): Promise<() => Promise> { - return lock(lockPath, { - lockfilePath: lockPath, - realpath: false, - stale: 10_000, - update: 3_000, - retries: 0 - }) +/** The acquisition gate remains held while retrying a Windows mkdir sharing failure. */ +export async function acquireTargetLock(lockPath: string, acquire: typeof lock = lock, + platform: NodeJS.Platform = process.platform): Promise<() => Promise> { + const delays = [20, 40, 80, 160, 320] + for (let attempt = 0; ; attempt++) { + try { + return await acquire(lockPath, { + lockfilePath: lockPath, + realpath: false, + stale: 10_000, + update: 3_000, + retries: 0 + }) + } catch (error) { + const failure = error as NodeJS.ErrnoException + const delay = delays[attempt] + if (platform !== 'win32' || !['EPERM', 'EBUSY'].includes(failure.code ?? '') + || failure.syscall !== 'mkdir' || failure.path !== lockPath || delay === undefined) throw error + await new Promise(resolve => setTimeout(resolve, delay)) + } + } } /** Serialize stale-lock recovery attempts without replacing a live contender. */ diff --git a/cli/test/unit/integrations/native-toggle.test.ts b/cli/test/unit/integrations/native-toggle.test.ts index f5b462f8a..52b9a7e5f 100644 --- a/cli/test/unit/integrations/native-toggle.test.ts +++ b/cli/test/unit/integrations/native-toggle.test.ts @@ -248,7 +248,10 @@ test('independent toggle transactions preserve both targets in shared inventory' })) return { operationId: accepted!.operationId, ownerId: 'owner', baseline, authorize: async () => {} } })) - expect((await Promise.all(requests.map(request => f.runtime.executeToggle(request)))).map(result => result.outcome)).toEqual(['committed', 'committed']) + const results = await Promise.allSettled(requests.map(request => f.runtime.executeToggle(request))) + const failures = results.filter(result => result.status === 'rejected') + if (failures.length) throw new AggregateError(failures.map(result => result.reason), 'Independent toggle transactions failed') + expect(results.map(result => result.status === 'fulfilled' ? result.value.outcome : undefined)).toEqual(['committed', 'committed']) expect((await f.inventory.read()).items[0]!.targets.map(target => target.installationState)).toEqual(['disabled', 'disabled']) } finally { await rm(f.home, { recursive: true, force: true }) } }) diff --git a/cli/test/unit/integrations/operation-store.test.ts b/cli/test/unit/integrations/operation-store.test.ts index 83ba95085..50d0c468e 100644 --- a/cli/test/unit/integrations/operation-store.test.ts +++ b/cli/test/unit/integrations/operation-store.test.ts @@ -1,6 +1,7 @@ +import { renameAtomicFile } from '../../../src/platform/atomic-rename' import { expect, test } from 'bun:test' import { randomUUID } from 'node:crypto' -import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readdir, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { OperationStore, operationRequestDigest } from '../../../src/integrations/operation-store' @@ -156,3 +157,61 @@ test.each(['identical', 'changed'] as const)('interrupted journal isolation hand expect(await readdir(root)).not.toContain(`${ordinary.operationId}.json`) } })) + + +test.each(['EPERM', 'EBUSY'])('journal replacement retries %s under the store lock without replaying validation or revision', async code => { + const root = await mkdtemp(join(tmpdir(), 'skillhub-journal-retry-')) + let armed = false; let attempts = 0; let validations = 0 + const reachedRename = Promise.withResolvers() + let firstAttempt: { source: string; target: string; payload: Buffer } | undefined + let previous: Buffer | undefined + let competing: Promise[]> | undefined + try { + const store = await OperationStore.open(root, (from, to) => renameAtomicFile(from, to, async (source, target) => { + if (armed) { + const observation = { source, target, payload: await readFile(source) } + if (!firstAttempt) firstAttempt = observation + else expect(observation).toEqual(firstAttempt) + expect(await readFile(target)).toEqual(previous!) + if (++attempts < 3) { reachedRename.resolve(); throw Object.assign(new Error('Sharing failure'), { code }) } + } + await rename(source, target) + }, 'win32')) + const key = request(store) + const initial = (await store.accept(key, async () => { validations++; return accepted }))! + const other = await OperationStore.open(root) + previous = await readFile(join(root, initial.operationId + '.json')) + armed = true + const updating = store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' }) + await reachedRename.promise + competing = Promise.allSettled([other.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })]) + const staged = await updating + expect(staged.revision).toBe(1) + expect((await competing)[0]!.status).toBe('rejected') + expect(attempts).toBe(3) + expect(validations).toBe(1) + expect((await store.get(initial.operationId, key.ownerId))!.revision).toBe(1) + expect((await store.accept({ ...key, mode: 'retry' }, async () => { throw new Error('must not replay') }))!.operationId).toBe(initial.operationId) + } finally { await competing; await rm(root, { recursive: true, force: true }) } +}) + +test('exhausted journal replacement preserves original bytes and releases the store lock', async () => { + const root = await mkdtemp(join(tmpdir(), 'skillhub-journal-exhaust-')) + let armed = false; let attempts = 0 + const failure = Object.assign(new Error('Sharing failure'), { code: 'EPERM' }) + try { + const store = await OperationStore.open(root, (from, to) => renameAtomicFile(from, to, async (source, target) => { + if (armed) { attempts++; throw failure } + await rename(source, target) + }, 'win32')) + const key = request(store); const initial = (await store.accept(key, async () => accepted))! + const path = join(root, initial.operationId + '.json'); const before = await readFile(path) + armed = true + await expect(store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })).rejects.toBe(failure) + expect(attempts).toBe(6) + expect(await readFile(path)).toEqual(before) + expect((await readdir(root)).filter(name => name.endsWith('.tmp'))).toEqual([]) + armed = false + expect((await store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })).revision).toBe(1) + } finally { await rm(root, { recursive: true, force: true }) } +}) diff --git a/cli/test/unit/services/skill-target-lock.test.ts b/cli/test/unit/services/skill-target-lock.test.ts index 99964f6f3..5802a2b3a 100644 --- a/cli/test/unit/services/skill-target-lock.test.ts +++ b/cli/test/unit/services/skill-target-lock.test.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url' import { describe, expect, test } from 'bun:test' import { acquireSkillTargetLock, + acquireTargetLock, assertPrivateLockDir, ensurePrivateLockDir, skillTargetLockPath, @@ -236,3 +237,71 @@ describe('skill target lifecycle lock', () => { expect(await exists(lockPathBeforeRemoval)).toBe(false) }) }) + + +test.each(['EPERM', 'EBUSY'])('Windows target lock retries only a transient mkdir %s on the selected lock', async code => { + const path = join(tmpdir(), 'skillhub-injected-lock'); let calls = 0 + const release = async () => {} + expect(await acquireTargetLock(path, async (target, options) => { + expect(target).toBe(path) + expect(options).toEqual({ lockfilePath: path, realpath: false, stale: 10000, update: 3000, retries: 0 }) + if (++calls < 3) throw Object.assign(new Error('Sharing failure'), { code, syscall: 'mkdir', path }) + return release + }, 'win32')).toBe(release) + expect(calls).toBe(3) +}) + +test('Windows target lock exhaustion preserves the original mkdir error', async () => { + const path = join(tmpdir(), 'skillhub-injected-lock') + const error = Object.assign(new Error('Sharing failure'), { code: 'EPERM', syscall: 'mkdir', path }); let calls = 0 + await expect(acquireTargetLock(path, async () => { calls++; throw error }, 'win32')).rejects.toBe(error) + expect(calls).toBe(6) +}) + +test.each([ + ['linux', 'EPERM', 'mkdir', true], ['darwin', 'EBUSY', 'mkdir', true], + ['win32', 'ELOCKED', 'mkdir', true], ['win32', 'EACCES', 'mkdir', true], + ['win32', 'EPERM', 'rmdir', true], ['win32', 'EPERM', 'mkdir', false], +] as const)('target lock rejects %s %s %s without broadening retry ownership', async (platform, code, syscall, selectedPath) => { + const path = join(tmpdir(), 'skillhub-injected-lock') + const error = Object.assign(new Error('No retry'), { code, syscall, path: selectedPath ? path : path + '-other' }); let calls = 0 + await expect(acquireTargetLock(path, async () => { calls++; throw error }, platform)).rejects.toBe(error) + expect(calls).toBe(1) +}) + +test('Windows target lock retries never steal a live proper-lockfile owner', async () => { + const root = await mkdtemp(join(tmpdir(), 'skillhub-lock-live-owner-')) + const path = join(root, 'target.lock'); const release = await acquireTargetLock(path) + try { + const original = await lstat(path) + await expect(acquireTargetLock(path, undefined, 'win32')).rejects.toMatchObject({ code: 'ELOCKED' }) + expect((await lstat(path)).mtimeMs).toBe(original.mtimeMs) + } finally { await release(); await rm(root, { recursive: true, force: true }) } +}) + + +test('target acquisition gate excludes another contender throughout Windows retry backoff', async () => { + const root = await mkdtemp(join(tmpdir(), 'skillhub-lock-retry-gate-')) + let secondEntered = false; let attempts = 0 + let second: Promise Promise>[]> | undefined + let release: (() => Promise) | undefined + try { + release = await acquireSkillTargetLock(root, 'writer', path => acquireTargetLock(path, async () => { + if (++attempts === 1) { + second = Promise.allSettled([acquireSkillTargetLock(root, 'writer', async otherPath => { + secondEntered = true; return acquireTargetLock(otherPath) + })]) + throw Object.assign(new Error('Sharing failure'), { code: 'EPERM', syscall: 'mkdir', path }) + } + expect(secondEntered).toBe(false) + return acquireTargetLock(path) + }, 'win32')) + const result = await second! + expect(attempts).toBe(2) + expect(secondEntered).toBe(true) + expect(result[0]!.status).toBe('rejected') + } finally { + if (second) for (const result of await second) if (result.status === 'fulfilled') await result.value() + await release?.(); await rm(root, { recursive: true, force: true }) + } +})