fix(dsh): tolerate Windows lock and journal sharing failures

Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
XiaoSeS 2026-10-29 19:12:41 +08:00
parent 8b54c2c07c
commit 4ebc1197ec
5 changed files with 173 additions and 26 deletions

View file

@ -1,9 +1,10 @@
import { renameAtomicFile } from '../platform/atomic-rename'
import { inventoryWithoutState, type Inventory } from '../stores/inventory-store'
import { isDeepStrictEqual } from 'node:util'
import { toggleJournal, validateToggleJournal, type ToggleJournal } from './toggle-journal'
import { createHash, randomUUID } from 'node:crypto'
import { constants } from 'node:fs'
import { chmod, link, lstat, mkdir, open, readdir, readFile, realpath, rename, rm, unlink } from 'node:fs/promises'
import { chmod, link, lstat, mkdir, open, readdir, readFile, realpath, rm, unlink } from 'node:fs/promises'
import { basename, dirname, isAbsolute, join } from 'node:path'
import { canonicalizeExistingPath, pathsOverlap } from '../platform/paths'
import { z } from 'zod'
@ -53,21 +54,22 @@ export function operationRequestDigest(parameters: Record<string, string | numbe
/** Host-only durable operation ownership. This store never executes or replays a file mutation. */
export class OperationStore {
private constructor(readonly root: string, readonly hostInstanceId: string) {}
private constructor(readonly root: string, readonly hostInstanceId: string,
private readonly replaceFile: typeof renameAtomicFile = renameAtomicFile) {}
static async open(root: string): Promise<OperationStore> {
static async open(root: string, replaceFile: typeof renameAtomicFile = renameAtomicFile): Promise<OperationStore> {
if (!isAbsolute(root)) throw new Error('Operation state requires a trusted absolute directory')
await mkdir(root, { recursive: true, mode: 0o700 })
const info = await lstat(root)
if (!info.isDirectory() || info.isSymbolicLink() || (process.getuid && info.uid !== process.getuid())) throw new Error('Unsafe operation state directory')
if (process.platform !== 'win32') await chmod(root, 0o700)
const pending = new OperationStore(root, '')
const pending = new OperationStore(root, '', replaceFile)
return pending.locked(async () => {
const path = join(root, 'host.json')
const saved = await readJson(path)
const hostInstanceId = saved === undefined ? randomUUID() : z.object({ hostInstanceId: id }).strict().parse(saved).hostInstanceId
if (saved === undefined) await atomicWrite(path, { hostInstanceId })
const store = new OperationStore(root, hostInstanceId)
if (saved === undefined) await atomicWrite(path, { hostInstanceId }, replaceFile)
const store = new OperationStore(root, hostInstanceId, replaceFile)
await store.isolateExtendedRecords()
return store
})
@ -483,7 +485,7 @@ export class OperationStore {
if (extended) {
await this.prepareExtendedDirectory()
await this.isolateRecord(value.operationId)
await writeOperationRecord(this.extendedPath(value.operationId), { ...value, schemaVersion: 3 })
await writeOperationRecord(this.extendedPath(value.operationId), { ...value, schemaVersion: 3 }, this.replaceFile)
return
}
// v2 snapshots contain ownership only, exactly as consumed by published clients.
@ -500,7 +502,7 @@ export class OperationStore {
compatible.removal.inventoryCommit.after = inventoryWithoutState(compatible.removal.inventoryCommit.after)
}
}
await writeOperationRecord(this.path(value.operationId), compatible)
await writeOperationRecord(this.path(value.operationId), compatible, this.replaceFile)
}
private async locked<T>(run: () => Promise<T>): Promise<T> {
@ -570,14 +572,14 @@ async function readJson(path: string): Promise<unknown | undefined> {
} finally { await file.close() }
}
async function atomicWrite(path: string, value: unknown): Promise<void> {
async function atomicWrite(path: string, value: unknown, replaceFile: typeof renameAtomicFile): Promise<void> {
const temporary = `${path}.${randomUUID()}.tmp`
try {
const payload = JSON.stringify(value)
if (Buffer.byteLength(payload) > 1024 * 1024) throw new OperationStoreError('CORRUPT_RECORD')
const file = await open(temporary, 'wx', 0o600)
try { await file.writeFile(payload); await file.sync() } finally { await file.close() }
await rename(temporary, path)
await replaceFile(temporary, path)
if (process.platform !== 'win32') {
const directory = await open(dirname(path), 'r')
try { await directory.sync() } finally { await directory.close() }
@ -598,10 +600,10 @@ async function syncJournalDirectory(path: string): Promise<void> {
}
/** Recovery may advance v1 status, but never rewrite the original serialized evidence. */
async function writeOperationRecord(path: string, value: OperationRecord): Promise<void> {
if (value.schemaVersion === 3) return atomicWrite(path, value)
async function writeOperationRecord(path: string, value: OperationRecord, replaceFile: typeof renameAtomicFile): Promise<void> {
if (value.schemaVersion === 3) return atomicWrite(path, value, replaceFile)
const previous = await readJson(path) as OperationRecord | undefined
if (!previous && value.schemaVersion === 2) return atomicWrite(path, value)
if (!previous && value.schemaVersion === 2) return atomicWrite(path, value, replaceFile)
if (!previous || previous.schemaVersion !== value.schemaVersion) throw new Error('Legacy operation evidence missing')
const preserved = structuredClone(value)
if (preserved.transaction && previous.transaction) {
@ -620,7 +622,7 @@ async function writeOperationRecord(path: string, value: OperationRecord): Promi
preserved.removal.inventoryCommit.after = previous.removal.inventoryCommit.after
}
}
await atomicWrite(path, preserved)
await atomicWrite(path, preserved, replaceFile)
}
/** A pending install may not exist yet; resolve the nearest existing ancestor. */

View file

@ -21,7 +21,8 @@ export async function orderSkillTargetLocks<T extends { rootDir: string }>(targe
}
/** Serializes every local lifecycle mutation for one Skill target directory. */
export async function acquireSkillTargetLock(rootDir: string, slug: string): Promise<() => Promise<void>> {
export async function acquireSkillTargetLock(rootDir: string, slug: string,
acquire: typeof acquireTargetLock = acquireTargetLock): Promise<() => Promise<void>> {
const lockPath = await skillTargetLockPath(rootDir, slug)
// proper-lockfile's stale deletion is not serialized. Gate only the acquisition attempt so two
// recoverers cannot remove and replace the same target lock concurrently.
@ -36,7 +37,7 @@ export async function acquireSkillTargetLock(rootDir: string, slug: string): Pro
let releaseTarget: () => Promise<void>
try {
releaseTarget = await acquireTargetLock(lockPath)
releaseTarget = await acquire(lockPath)
} catch (operationError) {
try {
await releaseAcquisitionGate()
@ -61,14 +62,27 @@ export async function acquireSkillTargetLock(rootDir: string, slug: string): Pro
return releaseTarget
}
function acquireTargetLock(lockPath: string): Promise<() => Promise<void>> {
return lock(lockPath, {
lockfilePath: lockPath,
realpath: false,
stale: 10_000,
update: 3_000,
retries: 0
})
/** The acquisition gate remains held while retrying a Windows mkdir sharing failure. */
export async function acquireTargetLock(lockPath: string, acquire: typeof lock = lock,
platform: NodeJS.Platform = process.platform): Promise<() => Promise<void>> {
const delays = [20, 40, 80, 160, 320]
for (let attempt = 0; ; attempt++) {
try {
return await acquire(lockPath, {
lockfilePath: lockPath,
realpath: false,
stale: 10_000,
update: 3_000,
retries: 0
})
} catch (error) {
const failure = error as NodeJS.ErrnoException
const delay = delays[attempt]
if (platform !== 'win32' || !['EPERM', 'EBUSY'].includes(failure.code ?? '')
|| failure.syscall !== 'mkdir' || failure.path !== lockPath || delay === undefined) throw error
await new Promise(resolve => setTimeout(resolve, delay))
}
}
}
/** Serialize stale-lock recovery attempts without replacing a live contender. */

View file

@ -248,7 +248,10 @@ test('independent toggle transactions preserve both targets in shared inventory'
}))
return { operationId: accepted!.operationId, ownerId: 'owner', baseline, authorize: async () => {} }
}))
expect((await Promise.all(requests.map(request => f.runtime.executeToggle(request)))).map(result => result.outcome)).toEqual(['committed', 'committed'])
const results = await Promise.allSettled(requests.map(request => f.runtime.executeToggle(request)))
const failures = results.filter(result => result.status === 'rejected')
if (failures.length) throw new AggregateError(failures.map(result => result.reason), 'Independent toggle transactions failed')
expect(results.map(result => result.status === 'fulfilled' ? result.value.outcome : undefined)).toEqual(['committed', 'committed'])
expect((await f.inventory.read()).items[0]!.targets.map(target => target.installationState)).toEqual(['disabled', 'disabled'])
} finally { await rm(f.home, { recursive: true, force: true }) }
})

View file

@ -1,6 +1,7 @@
import { renameAtomicFile } from '../../../src/platform/atomic-rename'
import { expect, test } from 'bun:test'
import { randomUUID } from 'node:crypto'
import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'
import { mkdir, mkdtemp, readdir, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { OperationStore, operationRequestDigest } from '../../../src/integrations/operation-store'
@ -156,3 +157,61 @@ test.each(['identical', 'changed'] as const)('interrupted journal isolation hand
expect(await readdir(root)).not.toContain(`${ordinary.operationId}.json`)
}
}))
test.each(['EPERM', 'EBUSY'])('journal replacement retries %s under the store lock without replaying validation or revision', async code => {
const root = await mkdtemp(join(tmpdir(), 'skillhub-journal-retry-'))
let armed = false; let attempts = 0; let validations = 0
const reachedRename = Promise.withResolvers<void>()
let firstAttempt: { source: string; target: string; payload: Buffer } | undefined
let previous: Buffer | undefined
let competing: Promise<PromiseSettledResult<unknown>[]> | undefined
try {
const store = await OperationStore.open(root, (from, to) => renameAtomicFile(from, to, async (source, target) => {
if (armed) {
const observation = { source, target, payload: await readFile(source) }
if (!firstAttempt) firstAttempt = observation
else expect(observation).toEqual(firstAttempt)
expect(await readFile(target)).toEqual(previous!)
if (++attempts < 3) { reachedRename.resolve(); throw Object.assign(new Error('Sharing failure'), { code }) }
}
await rename(source, target)
}, 'win32'))
const key = request(store)
const initial = (await store.accept(key, async () => { validations++; return accepted }))!
const other = await OperationStore.open(root)
previous = await readFile(join(root, initial.operationId + '.json'))
armed = true
const updating = store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })
await reachedRename.promise
competing = Promise.allSettled([other.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })])
const staged = await updating
expect(staged.revision).toBe(1)
expect((await competing)[0]!.status).toBe('rejected')
expect(attempts).toBe(3)
expect(validations).toBe(1)
expect((await store.get(initial.operationId, key.ownerId))!.revision).toBe(1)
expect((await store.accept({ ...key, mode: 'retry' }, async () => { throw new Error('must not replay') }))!.operationId).toBe(initial.operationId)
} finally { await competing; await rm(root, { recursive: true, force: true }) }
})
test('exhausted journal replacement preserves original bytes and releases the store lock', async () => {
const root = await mkdtemp(join(tmpdir(), 'skillhub-journal-exhaust-'))
let armed = false; let attempts = 0
const failure = Object.assign(new Error('Sharing failure'), { code: 'EPERM' })
try {
const store = await OperationStore.open(root, (from, to) => renameAtomicFile(from, to, async (source, target) => {
if (armed) { attempts++; throw failure }
await rename(source, target)
}, 'win32'))
const key = request(store); const initial = (await store.accept(key, async () => accepted))!
const path = join(root, initial.operationId + '.json'); const before = await readFile(path)
armed = true
await expect(store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })).rejects.toBe(failure)
expect(attempts).toBe(6)
expect(await readFile(path)).toEqual(before)
expect((await readdir(root)).filter(name => name.endsWith('.tmp'))).toEqual([])
armed = false
expect((await store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })).revision).toBe(1)
} finally { await rm(root, { recursive: true, force: true }) }
})

View file

@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'
import { describe, expect, test } from 'bun:test'
import {
acquireSkillTargetLock,
acquireTargetLock,
assertPrivateLockDir,
ensurePrivateLockDir,
skillTargetLockPath,
@ -236,3 +237,71 @@ describe('skill target lifecycle lock', () => {
expect(await exists(lockPathBeforeRemoval)).toBe(false)
})
})
test.each(['EPERM', 'EBUSY'])('Windows target lock retries only a transient mkdir %s on the selected lock', async code => {
const path = join(tmpdir(), 'skillhub-injected-lock'); let calls = 0
const release = async () => {}
expect(await acquireTargetLock(path, async (target, options) => {
expect(target).toBe(path)
expect(options).toEqual({ lockfilePath: path, realpath: false, stale: 10000, update: 3000, retries: 0 })
if (++calls < 3) throw Object.assign(new Error('Sharing failure'), { code, syscall: 'mkdir', path })
return release
}, 'win32')).toBe(release)
expect(calls).toBe(3)
})
test('Windows target lock exhaustion preserves the original mkdir error', async () => {
const path = join(tmpdir(), 'skillhub-injected-lock')
const error = Object.assign(new Error('Sharing failure'), { code: 'EPERM', syscall: 'mkdir', path }); let calls = 0
await expect(acquireTargetLock(path, async () => { calls++; throw error }, 'win32')).rejects.toBe(error)
expect(calls).toBe(6)
})
test.each([
['linux', 'EPERM', 'mkdir', true], ['darwin', 'EBUSY', 'mkdir', true],
['win32', 'ELOCKED', 'mkdir', true], ['win32', 'EACCES', 'mkdir', true],
['win32', 'EPERM', 'rmdir', true], ['win32', 'EPERM', 'mkdir', false],
] as const)('target lock rejects %s %s %s without broadening retry ownership', async (platform, code, syscall, selectedPath) => {
const path = join(tmpdir(), 'skillhub-injected-lock')
const error = Object.assign(new Error('No retry'), { code, syscall, path: selectedPath ? path : path + '-other' }); let calls = 0
await expect(acquireTargetLock(path, async () => { calls++; throw error }, platform)).rejects.toBe(error)
expect(calls).toBe(1)
})
test('Windows target lock retries never steal a live proper-lockfile owner', async () => {
const root = await mkdtemp(join(tmpdir(), 'skillhub-lock-live-owner-'))
const path = join(root, 'target.lock'); const release = await acquireTargetLock(path)
try {
const original = await lstat(path)
await expect(acquireTargetLock(path, undefined, 'win32')).rejects.toMatchObject({ code: 'ELOCKED' })
expect((await lstat(path)).mtimeMs).toBe(original.mtimeMs)
} finally { await release(); await rm(root, { recursive: true, force: true }) }
})
test('target acquisition gate excludes another contender throughout Windows retry backoff', async () => {
const root = await mkdtemp(join(tmpdir(), 'skillhub-lock-retry-gate-'))
let secondEntered = false; let attempts = 0
let second: Promise<PromiseSettledResult<() => Promise<void>>[]> | undefined
let release: (() => Promise<void>) | undefined
try {
release = await acquireSkillTargetLock(root, 'writer', path => acquireTargetLock(path, async () => {
if (++attempts === 1) {
second = Promise.allSettled([acquireSkillTargetLock(root, 'writer', async otherPath => {
secondEntered = true; return acquireTargetLock(otherPath)
})])
throw Object.assign(new Error('Sharing failure'), { code: 'EPERM', syscall: 'mkdir', path })
}
expect(secondEntered).toBe(false)
return acquireTargetLock(path)
}, 'win32'))
const result = await second!
expect(attempts).toBe(2)
expect(secondEntered).toBe(true)
expect(result[0]!.status).toBe('rejected')
} finally {
if (second) for (const result of await second) if (result.status === 'fulfilled') await result.value()
await release?.(); await rm(root, { recursive: true, force: true })
}
})