mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-16 09:21:18 +00:00
fix(dsh): tolerate Windows lock and journal sharing failures
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
parent
8b54c2c07c
commit
4ebc1197ec
5 changed files with 173 additions and 26 deletions
|
|
@ -1,9 +1,10 @@
|
|||
import { renameAtomicFile } from '../platform/atomic-rename'
|
||||
import { inventoryWithoutState, type Inventory } from '../stores/inventory-store'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import { toggleJournal, validateToggleJournal, type ToggleJournal } from './toggle-journal'
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { constants } from 'node:fs'
|
||||
import { chmod, link, lstat, mkdir, open, readdir, readFile, realpath, rename, rm, unlink } from 'node:fs/promises'
|
||||
import { chmod, link, lstat, mkdir, open, readdir, readFile, realpath, rm, unlink } from 'node:fs/promises'
|
||||
import { basename, dirname, isAbsolute, join } from 'node:path'
|
||||
import { canonicalizeExistingPath, pathsOverlap } from '../platform/paths'
|
||||
import { z } from 'zod'
|
||||
|
|
@ -53,21 +54,22 @@ export function operationRequestDigest(parameters: Record<string, string | numbe
|
|||
|
||||
/** Host-only durable operation ownership. This store never executes or replays a file mutation. */
|
||||
export class OperationStore {
|
||||
private constructor(readonly root: string, readonly hostInstanceId: string) {}
|
||||
private constructor(readonly root: string, readonly hostInstanceId: string,
|
||||
private readonly replaceFile: typeof renameAtomicFile = renameAtomicFile) {}
|
||||
|
||||
static async open(root: string): Promise<OperationStore> {
|
||||
static async open(root: string, replaceFile: typeof renameAtomicFile = renameAtomicFile): Promise<OperationStore> {
|
||||
if (!isAbsolute(root)) throw new Error('Operation state requires a trusted absolute directory')
|
||||
await mkdir(root, { recursive: true, mode: 0o700 })
|
||||
const info = await lstat(root)
|
||||
if (!info.isDirectory() || info.isSymbolicLink() || (process.getuid && info.uid !== process.getuid())) throw new Error('Unsafe operation state directory')
|
||||
if (process.platform !== 'win32') await chmod(root, 0o700)
|
||||
const pending = new OperationStore(root, '')
|
||||
const pending = new OperationStore(root, '', replaceFile)
|
||||
return pending.locked(async () => {
|
||||
const path = join(root, 'host.json')
|
||||
const saved = await readJson(path)
|
||||
const hostInstanceId = saved === undefined ? randomUUID() : z.object({ hostInstanceId: id }).strict().parse(saved).hostInstanceId
|
||||
if (saved === undefined) await atomicWrite(path, { hostInstanceId })
|
||||
const store = new OperationStore(root, hostInstanceId)
|
||||
if (saved === undefined) await atomicWrite(path, { hostInstanceId }, replaceFile)
|
||||
const store = new OperationStore(root, hostInstanceId, replaceFile)
|
||||
await store.isolateExtendedRecords()
|
||||
return store
|
||||
})
|
||||
|
|
@ -483,7 +485,7 @@ export class OperationStore {
|
|||
if (extended) {
|
||||
await this.prepareExtendedDirectory()
|
||||
await this.isolateRecord(value.operationId)
|
||||
await writeOperationRecord(this.extendedPath(value.operationId), { ...value, schemaVersion: 3 })
|
||||
await writeOperationRecord(this.extendedPath(value.operationId), { ...value, schemaVersion: 3 }, this.replaceFile)
|
||||
return
|
||||
}
|
||||
// v2 snapshots contain ownership only, exactly as consumed by published clients.
|
||||
|
|
@ -500,7 +502,7 @@ export class OperationStore {
|
|||
compatible.removal.inventoryCommit.after = inventoryWithoutState(compatible.removal.inventoryCommit.after)
|
||||
}
|
||||
}
|
||||
await writeOperationRecord(this.path(value.operationId), compatible)
|
||||
await writeOperationRecord(this.path(value.operationId), compatible, this.replaceFile)
|
||||
}
|
||||
|
||||
private async locked<T>(run: () => Promise<T>): Promise<T> {
|
||||
|
|
@ -570,14 +572,14 @@ async function readJson(path: string): Promise<unknown | undefined> {
|
|||
} finally { await file.close() }
|
||||
}
|
||||
|
||||
async function atomicWrite(path: string, value: unknown): Promise<void> {
|
||||
async function atomicWrite(path: string, value: unknown, replaceFile: typeof renameAtomicFile): Promise<void> {
|
||||
const temporary = `${path}.${randomUUID()}.tmp`
|
||||
try {
|
||||
const payload = JSON.stringify(value)
|
||||
if (Buffer.byteLength(payload) > 1024 * 1024) throw new OperationStoreError('CORRUPT_RECORD')
|
||||
const file = await open(temporary, 'wx', 0o600)
|
||||
try { await file.writeFile(payload); await file.sync() } finally { await file.close() }
|
||||
await rename(temporary, path)
|
||||
await replaceFile(temporary, path)
|
||||
if (process.platform !== 'win32') {
|
||||
const directory = await open(dirname(path), 'r')
|
||||
try { await directory.sync() } finally { await directory.close() }
|
||||
|
|
@ -598,10 +600,10 @@ async function syncJournalDirectory(path: string): Promise<void> {
|
|||
}
|
||||
|
||||
/** Recovery may advance v1 status, but never rewrite the original serialized evidence. */
|
||||
async function writeOperationRecord(path: string, value: OperationRecord): Promise<void> {
|
||||
if (value.schemaVersion === 3) return atomicWrite(path, value)
|
||||
async function writeOperationRecord(path: string, value: OperationRecord, replaceFile: typeof renameAtomicFile): Promise<void> {
|
||||
if (value.schemaVersion === 3) return atomicWrite(path, value, replaceFile)
|
||||
const previous = await readJson(path) as OperationRecord | undefined
|
||||
if (!previous && value.schemaVersion === 2) return atomicWrite(path, value)
|
||||
if (!previous && value.schemaVersion === 2) return atomicWrite(path, value, replaceFile)
|
||||
if (!previous || previous.schemaVersion !== value.schemaVersion) throw new Error('Legacy operation evidence missing')
|
||||
const preserved = structuredClone(value)
|
||||
if (preserved.transaction && previous.transaction) {
|
||||
|
|
@ -620,7 +622,7 @@ async function writeOperationRecord(path: string, value: OperationRecord): Promi
|
|||
preserved.removal.inventoryCommit.after = previous.removal.inventoryCommit.after
|
||||
}
|
||||
}
|
||||
await atomicWrite(path, preserved)
|
||||
await atomicWrite(path, preserved, replaceFile)
|
||||
}
|
||||
|
||||
/** A pending install may not exist yet; resolve the nearest existing ancestor. */
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ export async function orderSkillTargetLocks<T extends { rootDir: string }>(targe
|
|||
}
|
||||
|
||||
/** Serializes every local lifecycle mutation for one Skill target directory. */
|
||||
export async function acquireSkillTargetLock(rootDir: string, slug: string): Promise<() => Promise<void>> {
|
||||
export async function acquireSkillTargetLock(rootDir: string, slug: string,
|
||||
acquire: typeof acquireTargetLock = acquireTargetLock): Promise<() => Promise<void>> {
|
||||
const lockPath = await skillTargetLockPath(rootDir, slug)
|
||||
// proper-lockfile's stale deletion is not serialized. Gate only the acquisition attempt so two
|
||||
// recoverers cannot remove and replace the same target lock concurrently.
|
||||
|
|
@ -36,7 +37,7 @@ export async function acquireSkillTargetLock(rootDir: string, slug: string): Pro
|
|||
|
||||
let releaseTarget: () => Promise<void>
|
||||
try {
|
||||
releaseTarget = await acquireTargetLock(lockPath)
|
||||
releaseTarget = await acquire(lockPath)
|
||||
} catch (operationError) {
|
||||
try {
|
||||
await releaseAcquisitionGate()
|
||||
|
|
@ -61,14 +62,27 @@ export async function acquireSkillTargetLock(rootDir: string, slug: string): Pro
|
|||
return releaseTarget
|
||||
}
|
||||
|
||||
function acquireTargetLock(lockPath: string): Promise<() => Promise<void>> {
|
||||
return lock(lockPath, {
|
||||
lockfilePath: lockPath,
|
||||
realpath: false,
|
||||
stale: 10_000,
|
||||
update: 3_000,
|
||||
retries: 0
|
||||
})
|
||||
/** The acquisition gate remains held while retrying a Windows mkdir sharing failure. */
|
||||
export async function acquireTargetLock(lockPath: string, acquire: typeof lock = lock,
|
||||
platform: NodeJS.Platform = process.platform): Promise<() => Promise<void>> {
|
||||
const delays = [20, 40, 80, 160, 320]
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await acquire(lockPath, {
|
||||
lockfilePath: lockPath,
|
||||
realpath: false,
|
||||
stale: 10_000,
|
||||
update: 3_000,
|
||||
retries: 0
|
||||
})
|
||||
} catch (error) {
|
||||
const failure = error as NodeJS.ErrnoException
|
||||
const delay = delays[attempt]
|
||||
if (platform !== 'win32' || !['EPERM', 'EBUSY'].includes(failure.code ?? '')
|
||||
|| failure.syscall !== 'mkdir' || failure.path !== lockPath || delay === undefined) throw error
|
||||
await new Promise(resolve => setTimeout(resolve, delay))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Serialize stale-lock recovery attempts without replacing a live contender. */
|
||||
|
|
|
|||
|
|
@ -248,7 +248,10 @@ test('independent toggle transactions preserve both targets in shared inventory'
|
|||
}))
|
||||
return { operationId: accepted!.operationId, ownerId: 'owner', baseline, authorize: async () => {} }
|
||||
}))
|
||||
expect((await Promise.all(requests.map(request => f.runtime.executeToggle(request)))).map(result => result.outcome)).toEqual(['committed', 'committed'])
|
||||
const results = await Promise.allSettled(requests.map(request => f.runtime.executeToggle(request)))
|
||||
const failures = results.filter(result => result.status === 'rejected')
|
||||
if (failures.length) throw new AggregateError(failures.map(result => result.reason), 'Independent toggle transactions failed')
|
||||
expect(results.map(result => result.status === 'fulfilled' ? result.value.outcome : undefined)).toEqual(['committed', 'committed'])
|
||||
expect((await f.inventory.read()).items[0]!.targets.map(target => target.installationState)).toEqual(['disabled', 'disabled'])
|
||||
} finally { await rm(f.home, { recursive: true, force: true }) }
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { renameAtomicFile } from '../../../src/platform/atomic-rename'
|
||||
import { expect, test } from 'bun:test'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'
|
||||
import { mkdir, mkdtemp, readdir, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { OperationStore, operationRequestDigest } from '../../../src/integrations/operation-store'
|
||||
|
|
@ -156,3 +157,61 @@ test.each(['identical', 'changed'] as const)('interrupted journal isolation hand
|
|||
expect(await readdir(root)).not.toContain(`${ordinary.operationId}.json`)
|
||||
}
|
||||
}))
|
||||
|
||||
|
||||
test.each(['EPERM', 'EBUSY'])('journal replacement retries %s under the store lock without replaying validation or revision', async code => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'skillhub-journal-retry-'))
|
||||
let armed = false; let attempts = 0; let validations = 0
|
||||
const reachedRename = Promise.withResolvers<void>()
|
||||
let firstAttempt: { source: string; target: string; payload: Buffer } | undefined
|
||||
let previous: Buffer | undefined
|
||||
let competing: Promise<PromiseSettledResult<unknown>[]> | undefined
|
||||
try {
|
||||
const store = await OperationStore.open(root, (from, to) => renameAtomicFile(from, to, async (source, target) => {
|
||||
if (armed) {
|
||||
const observation = { source, target, payload: await readFile(source) }
|
||||
if (!firstAttempt) firstAttempt = observation
|
||||
else expect(observation).toEqual(firstAttempt)
|
||||
expect(await readFile(target)).toEqual(previous!)
|
||||
if (++attempts < 3) { reachedRename.resolve(); throw Object.assign(new Error('Sharing failure'), { code }) }
|
||||
}
|
||||
await rename(source, target)
|
||||
}, 'win32'))
|
||||
const key = request(store)
|
||||
const initial = (await store.accept(key, async () => { validations++; return accepted }))!
|
||||
const other = await OperationStore.open(root)
|
||||
previous = await readFile(join(root, initial.operationId + '.json'))
|
||||
armed = true
|
||||
const updating = store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })
|
||||
await reachedRename.promise
|
||||
competing = Promise.allSettled([other.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })])
|
||||
const staged = await updating
|
||||
expect(staged.revision).toBe(1)
|
||||
expect((await competing)[0]!.status).toBe('rejected')
|
||||
expect(attempts).toBe(3)
|
||||
expect(validations).toBe(1)
|
||||
expect((await store.get(initial.operationId, key.ownerId))!.revision).toBe(1)
|
||||
expect((await store.accept({ ...key, mode: 'retry' }, async () => { throw new Error('must not replay') }))!.operationId).toBe(initial.operationId)
|
||||
} finally { await competing; await rm(root, { recursive: true, force: true }) }
|
||||
})
|
||||
|
||||
test('exhausted journal replacement preserves original bytes and releases the store lock', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'skillhub-journal-exhaust-'))
|
||||
let armed = false; let attempts = 0
|
||||
const failure = Object.assign(new Error('Sharing failure'), { code: 'EPERM' })
|
||||
try {
|
||||
const store = await OperationStore.open(root, (from, to) => renameAtomicFile(from, to, async (source, target) => {
|
||||
if (armed) { attempts++; throw failure }
|
||||
await rename(source, target)
|
||||
}, 'win32'))
|
||||
const key = request(store); const initial = (await store.accept(key, async () => accepted))!
|
||||
const path = join(root, initial.operationId + '.json'); const before = await readFile(path)
|
||||
armed = true
|
||||
await expect(store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })).rejects.toBe(failure)
|
||||
expect(attempts).toBe(6)
|
||||
expect(await readFile(path)).toEqual(before)
|
||||
expect((await readdir(root)).filter(name => name.endsWith('.tmp'))).toEqual([])
|
||||
armed = false
|
||||
expect((await store.update(initial.operationId, key.ownerId, 0, { ...initial.operation.status, phase: 'staging' })).revision).toBe(1)
|
||||
} finally { await rm(root, { recursive: true, force: true }) }
|
||||
})
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'
|
|||
import { describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSkillTargetLock,
|
||||
acquireTargetLock,
|
||||
assertPrivateLockDir,
|
||||
ensurePrivateLockDir,
|
||||
skillTargetLockPath,
|
||||
|
|
@ -236,3 +237,71 @@ describe('skill target lifecycle lock', () => {
|
|||
expect(await exists(lockPathBeforeRemoval)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
test.each(['EPERM', 'EBUSY'])('Windows target lock retries only a transient mkdir %s on the selected lock', async code => {
|
||||
const path = join(tmpdir(), 'skillhub-injected-lock'); let calls = 0
|
||||
const release = async () => {}
|
||||
expect(await acquireTargetLock(path, async (target, options) => {
|
||||
expect(target).toBe(path)
|
||||
expect(options).toEqual({ lockfilePath: path, realpath: false, stale: 10000, update: 3000, retries: 0 })
|
||||
if (++calls < 3) throw Object.assign(new Error('Sharing failure'), { code, syscall: 'mkdir', path })
|
||||
return release
|
||||
}, 'win32')).toBe(release)
|
||||
expect(calls).toBe(3)
|
||||
})
|
||||
|
||||
test('Windows target lock exhaustion preserves the original mkdir error', async () => {
|
||||
const path = join(tmpdir(), 'skillhub-injected-lock')
|
||||
const error = Object.assign(new Error('Sharing failure'), { code: 'EPERM', syscall: 'mkdir', path }); let calls = 0
|
||||
await expect(acquireTargetLock(path, async () => { calls++; throw error }, 'win32')).rejects.toBe(error)
|
||||
expect(calls).toBe(6)
|
||||
})
|
||||
|
||||
test.each([
|
||||
['linux', 'EPERM', 'mkdir', true], ['darwin', 'EBUSY', 'mkdir', true],
|
||||
['win32', 'ELOCKED', 'mkdir', true], ['win32', 'EACCES', 'mkdir', true],
|
||||
['win32', 'EPERM', 'rmdir', true], ['win32', 'EPERM', 'mkdir', false],
|
||||
] as const)('target lock rejects %s %s %s without broadening retry ownership', async (platform, code, syscall, selectedPath) => {
|
||||
const path = join(tmpdir(), 'skillhub-injected-lock')
|
||||
const error = Object.assign(new Error('No retry'), { code, syscall, path: selectedPath ? path : path + '-other' }); let calls = 0
|
||||
await expect(acquireTargetLock(path, async () => { calls++; throw error }, platform)).rejects.toBe(error)
|
||||
expect(calls).toBe(1)
|
||||
})
|
||||
|
||||
test('Windows target lock retries never steal a live proper-lockfile owner', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'skillhub-lock-live-owner-'))
|
||||
const path = join(root, 'target.lock'); const release = await acquireTargetLock(path)
|
||||
try {
|
||||
const original = await lstat(path)
|
||||
await expect(acquireTargetLock(path, undefined, 'win32')).rejects.toMatchObject({ code: 'ELOCKED' })
|
||||
expect((await lstat(path)).mtimeMs).toBe(original.mtimeMs)
|
||||
} finally { await release(); await rm(root, { recursive: true, force: true }) }
|
||||
})
|
||||
|
||||
|
||||
test('target acquisition gate excludes another contender throughout Windows retry backoff', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'skillhub-lock-retry-gate-'))
|
||||
let secondEntered = false; let attempts = 0
|
||||
let second: Promise<PromiseSettledResult<() => Promise<void>>[]> | undefined
|
||||
let release: (() => Promise<void>) | undefined
|
||||
try {
|
||||
release = await acquireSkillTargetLock(root, 'writer', path => acquireTargetLock(path, async () => {
|
||||
if (++attempts === 1) {
|
||||
second = Promise.allSettled([acquireSkillTargetLock(root, 'writer', async otherPath => {
|
||||
secondEntered = true; return acquireTargetLock(otherPath)
|
||||
})])
|
||||
throw Object.assign(new Error('Sharing failure'), { code: 'EPERM', syscall: 'mkdir', path })
|
||||
}
|
||||
expect(secondEntered).toBe(false)
|
||||
return acquireTargetLock(path)
|
||||
}, 'win32'))
|
||||
const result = await second!
|
||||
expect(attempts).toBe(2)
|
||||
expect(secondEntered).toBe(true)
|
||||
expect(result[0]!.status).toBe('rejected')
|
||||
} finally {
|
||||
if (second) for (const result of await second) if (result.status === 'fulfilled') await result.value()
|
||||
await release?.(); await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue