diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 175db47e6..47745c575 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -27,6 +27,7 @@ import { warnMissingOptionalGrammars } from './optional-grammars.js'; import { glob } from 'glob'; import fs from 'fs/promises'; import { cliError } from './cli-message.js'; +import { isHfDownloadFailure } from '../core/embeddings/hf-env.js'; // Capture stderr.write at module load BEFORE anything (LadybugDB native // init, progress bar, console redirection) can monkey-patch it. The @@ -576,6 +577,26 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption return; } + // HF download failure β€” show clean guidance without the raw stack trace. + // Checked before writeFatalToStderr so the user sees one focused message + // rather than a stack-trace dump followed by a second remediation block. + if (isHfDownloadFailure(msg) || msg.includes('Failed to download embedding model')) { + cliError( + ` The embedding model could not be downloaded.\n` + + ` huggingface.co may be unreachable from your network\n` + + ` (e.g. behind a corporate proxy or a regional firewall).\n` + + ` Suggestions:\n` + + ` 1. Set HF_ENDPOINT to a mirror and retry:\n` + + ` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` + + ` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)\n` + + ` 2. Check your proxy / VPN settings.\n` + + ` 3. Once downloaded the model is cached β€” future runs work offline.\n`, + { recoveryHint: 'hf-endpoint-unreachable' }, + ); + process.exitCode = 1; + return; + } + // Bypass the redirected console.error and write the full stack to // the real stderr captured at module load. The redirected // console.error wraps every line with `\\x1b[2K\\r` (ANSI clear-line) diff --git a/gitnexus/src/core/embeddings/embedder.ts b/gitnexus/src/core/embeddings/embedder.ts index b37fb45f3..72ddcbd70 100644 --- a/gitnexus/src/core/embeddings/embedder.ts +++ b/gitnexus/src/core/embeddings/embedder.ts @@ -14,7 +14,12 @@ if (!process.env.ORT_LOG_LEVEL) { process.env.ORT_LOG_LEVEL = '3'; } -import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers'; +import { + pipeline, + env, + type FeatureExtractionPipeline, + type ProgressInfo, +} from '@huggingface/transformers'; import { existsSync } from 'fs'; import { execFileSync } from 'child_process'; import { join, dirname } from 'path'; @@ -22,7 +27,7 @@ import { createRequire } from 'module'; import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js'; import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js'; import { resolveEmbeddingConfig } from './config.js'; -import { applyHfEnvOverrides } from './hf-env.js'; +import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js'; import { logger } from '../logger.js'; /** @@ -171,13 +176,18 @@ export const initEmbedder = async ( } const progressCallback = onProgress - ? (data: any) => { + ? (data: ProgressInfo) => { const progress: ModelProgress = { - status: data.status || 'progress', - file: data.file, - progress: data.progress, - loaded: data.loaded, - total: data.total, + // Map the `progress_total` aggregate event (not in ModelProgress.status) + // back to 'progress' so callers don't need to handle it separately. + status: + data.status === 'progress_total' + ? 'progress' + : ((data.status as ModelProgress['status']) ?? 'progress'), + file: 'file' in data ? data.file : undefined, + progress: 'progress' in data ? data.progress : undefined, + loaded: 'loaded' in data ? data.loaded : undefined, + total: 'total' in data ? data.total : undefined, }; onProgress(progress); } @@ -202,17 +212,29 @@ export const initEmbedder = async ( logger.info('πŸ”§ Using WASM backend (slower)...'); } - embedderInstance = await (pipeline as any)('feature-extraction', finalConfig.modelId, { - device: device, - dtype: 'fp32', - progress_callback: progressCallback, - session_options: { - logSeverityLevel: 3, - intraOpNumThreads: finalConfig.threads, - interOpNumThreads: 1, - executionMode: 'sequential', + embedderInstance = await withHfDownloadRetry( + () => + pipeline('feature-extraction', finalConfig.modelId, { + device: device, + dtype: 'fp32', + progress_callback: progressCallback, + session_options: { + logSeverityLevel: 3, + intraOpNumThreads: finalConfig.threads, + interOpNumThreads: 1, + executionMode: 'sequential', + }, + }), + { + onRetry: isDev + ? (attempt, max, err) => + logger.warn( + { attempt, max, err: err.message }, + `⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`, + ) + : undefined, }, - }); + ); currentDevice = device; if (isDev) { @@ -228,6 +250,20 @@ export const initEmbedder = async ( return embedderInstance!; } catch (deviceError) { + // Network errors and circuit-open errors are not device-specific β€” + // they will fail the same way on every device. Rethrow immediately + // with actionable HF_ENDPOINT guidance rather than silently falling + // back to the next device. + const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError); + if (isHfDownloadFailure(errMsg)) { + const endpointHint = process.env.HF_ENDPOINT + ? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.` + : `huggingface.co may be unreachable from your network.\n` + + ` Set HF_ENDPOINT to a mirror and retry:\n` + + ` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` + + ` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`; + throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`); + } if (isDev && (device === 'cuda' || device === 'dml')) { const gpuType = device === 'dml' ? 'DirectML' : 'CUDA'; logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`); diff --git a/gitnexus/src/core/embeddings/hf-env.ts b/gitnexus/src/core/embeddings/hf-env.ts index 6a977a76d..95548fff2 100644 --- a/gitnexus/src/core/embeddings/hf-env.ts +++ b/gitnexus/src/core/embeddings/hf-env.ts @@ -1,6 +1,25 @@ import os from 'node:os'; import { join } from 'node:path'; +// --------------------------------------------------------------------------- +// Download resilience defaults +// --------------------------------------------------------------------------- + +/** Per-attempt timeout for the full model download (5 minutes). */ +export const HF_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1_000; +/** Maximum total download attempts (1 initial + N-1 retries). */ +export const HF_MAX_ATTEMPTS = 3; +/** Initial delay between retry attempts; doubles on each subsequent retry. */ +export const HF_BASE_DELAY_MS = 2_000; +/** Number of consecutive failures required to open the circuit. */ +export const CB_FAILURE_THRESHOLD = 3; +/** How long the circuit stays open before transitioning to half-open. */ +export const CB_RESET_TIMEOUT_MS = 60_000; +/** Upper bound clamped on the env-override per-attempt timeout (30 minutes). */ +export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000; +/** Upper bound clamped on the env-override attempt count. */ +export const HF_MAX_ATTEMPTS_CAP = 10; + /** * @internal Exported only for unit tests and the two embedder entry points * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the @@ -60,3 +79,265 @@ export function applyHfEnvOverrides(env: HfEnvSubset): void { env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/'; } } + +/** + * @internal Exported for unit tests and the two embedder entry points. + * + * Returns true when an error message indicates a network-level fetch failure + * during HuggingFace model download (e.g. `TypeError: fetch failed`, + * `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`). + * + * These errors are not device-specific and cannot be fixed by falling back to + * a different ONNX device β€” the caller should rethrow immediately with + * guidance about `HF_ENDPOINT`. + */ +export function isNetworkFetchError(message: string): boolean { + return ( + message.includes('fetch failed') || + message.includes('ECONNREFUSED') || + message.includes('ENOTFOUND') || + message.includes('ETIMEDOUT') || + message.includes('ECONNRESET') + ); +} + +// --------------------------------------------------------------------------- +// Circuit breaker +// --------------------------------------------------------------------------- + +/** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */ +export const CIRCUIT_OPEN_TAG = 'hf-circuit-open'; + +/** Circuit-breaker states. */ +type CircuitState = 'closed' | 'open' | 'half-open'; + +/** + * Circuit breaker for HuggingFace model downloads. + * + * After `failureThreshold` consecutive network failures the circuit opens and + * all subsequent calls to `withHfDownloadRetry` fail immediately without + * issuing any network requests. After `resetTimeoutMs` the circuit enters the + * half-open state and the next call is attempted β€” if it succeeds the circuit + * closes again; if it fails the circuit re-opens. + * + * Exported for unit-testing; production code should use the module-level + * `hfDownloadCircuit` singleton. + */ +export class HfDownloadCircuitBreaker { + private _state: CircuitState = 'closed'; + private _failures = 0; + /** Timestamp of the last recorded failure (ms since epoch). */ + lastFailureAt = 0; + + constructor( + readonly failureThreshold: number = CB_FAILURE_THRESHOLD, + readonly resetTimeoutMs: number = CB_RESET_TIMEOUT_MS, + ) {} + + /** Effective state, factoring in the reset-timeout transition. */ + get state(): CircuitState { + if (this._state === 'open' && Date.now() - this.lastFailureAt > this.resetTimeoutMs) { + this._state = 'half-open'; + } + return this._state; + } + + /** Returns true when the circuit is open and calls should be rejected. */ + isOpen(): boolean { + return this.state === 'open'; + } + + /** Record a successful call β€” resets the failure counter and closes the circuit. */ + recordSuccess(): void { + this._failures = 0; + this._state = 'closed'; + } + + /** Record a failed call β€” increments the counter and opens the circuit when the threshold is reached. */ + recordFailure(): void { + this._failures++; + this.lastFailureAt = Date.now(); + if (this._failures >= this.failureThreshold) { + this._state = 'open'; + } + } + + /** @internal Reset to initial state (used in tests). */ + reset(): void { + this._failures = 0; + this._state = 'closed'; + this.lastFailureAt = 0; + } +} + +/** Module-level singleton shared by both embedder entry points. */ +export const hfDownloadCircuit = new HfDownloadCircuitBreaker(); + +// --------------------------------------------------------------------------- +// Retry + timeout wrapper +// --------------------------------------------------------------------------- + +/** @internal Returns true for errors that should abort without retry (circuit-open). */ +export function isHfCircuitOpenError(message: string): boolean { + return message.includes(CIRCUIT_OPEN_TAG); +} + +/** + * Returns true for any HuggingFace download failure that warrants showing the + * `HF_ENDPOINT` remediation hint: either a raw network error or a + * circuit-open rejection (which itself was caused by repeated network errors). + */ +export function isHfDownloadFailure(message: string): boolean { + return isNetworkFetchError(message) || isHfCircuitOpenError(message); +} + +/** @internal Wraps `fn` in a hard time-limit. The timeout error contains + * `ETIMEDOUT` so that `isNetworkFetchError` classifies it correctly. + */ +export function withDownloadTimeout(fn: () => Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => + reject( + new Error( + `ETIMEDOUT: model download timed out after ${Math.round(timeoutMs / 1000)}s β€” ` + + `check your network speed or set HF_ENDPOINT to a faster mirror`, + ), + ), + timeoutMs, + ); + fn().then( + (v) => { + clearTimeout(timer); + resolve(v); + }, + (e) => { + clearTimeout(timer); + reject(e); + }, + ); + }); +} + +/** @internal Async sleep (exposed for testing). */ +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export interface HfRetryOptions { + /** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */ + maxAttempts?: number; + /** Delay before the first retry; doubles on each subsequent attempt (default: `HF_BASE_DELAY_MS`). */ + baseDelayMs?: number; + /** Per-attempt wall-clock timeout in ms (default: `HF_DOWNLOAD_TIMEOUT_MS`). */ + timeoutMs?: number; + /** + * Circuit-breaker instance to use. Defaults to the module-level + * `hfDownloadCircuit` singleton. Pass a fresh instance in tests. + */ + circuit?: HfDownloadCircuitBreaker; + /** + * Optional callback invoked before each retry (not the initial attempt). + * @param attempt - 1-based retry number + * @param max - total allowed attempts + * @param error - the error that triggered the retry + */ + onRetry?: (attempt: number, max: number, error: Error) => void; +} + +/** + * Retry wrapper for HuggingFace model downloads with per-attempt timeout and + * circuit-breaker protection. + * + * Behaviour: + * - If the circuit is **open**, fails immediately with a `CIRCUIT_OPEN_TAG` + * message (so `isHfDownloadFailure` still returns true and the caller can + * show `HF_ENDPOINT` guidance). + * - Each attempt is wrapped in `withDownloadTimeout`. + * - On a network-level error (`isNetworkFetchError`) the attempt is retried + * with exponential back-off; non-network errors (e.g. ONNX device failure) + * are rethrown immediately without retry. + * - Every network failure is recorded on the circuit breaker; a success resets + * it. + * - After all attempts are exhausted, the last network error is rethrown + * so the existing `isNetworkFetchError` / `isHfDownloadFailure` guards in + * the calling code still fire. + */ +export async function withHfDownloadRetry( + fn: () => Promise, + options: HfRetryOptions = {}, +): Promise { + // Resolve effective values β€” explicit options take precedence over env vars, + // which take precedence over built-in defaults. This lets users lower the + // per-attempt timeout without rebuilding (e.g. + // HF_DOWNLOAD_TIMEOUT_MS=60000 npx gitnexus analyze --embeddings + // reduces the worst-case wait from 15 minutes to ~3 minutes). + // + // Upper bounds are clamped to prevent accidental runaway configuration: + // - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min) + // - maxAttempts is floored (fractional values β†’ integer) and capped at + // HF_MAX_ATTEMPTS_CAP (10). Values ≀ 0, NaN, or Infinity fall back to + // the built-in defaults. + const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS); + const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS); + const resolvedTimeout = + Number.isFinite(envTimeout) && envTimeout > 0 + ? Math.min(envTimeout, HF_MAX_TIMEOUT_MS) + : HF_DOWNLOAD_TIMEOUT_MS; + const resolvedMaxAttempts = + Number.isFinite(envMaxAttempts) && envMaxAttempts > 0 + ? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP) + : HF_MAX_ATTEMPTS; + const { + maxAttempts = resolvedMaxAttempts, + baseDelayMs = HF_BASE_DELAY_MS, + timeoutMs = resolvedTimeout, + circuit = hfDownloadCircuit, + onRetry, + } = options; + if (circuit.isOpen()) { + const secsUntilReset = Math.ceil( + (circuit.resetTimeoutMs - (Date.now() - circuit.lastFailureAt)) / 1000, + ); + throw new Error( + `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` + + (secsUntilReset > 0 ? ` β€” will reset in ~${secsUntilReset}s` : ''), + ); + } + + let lastError: Error = new Error('unknown error'); + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + const result = await withDownloadTimeout(fn, timeoutMs); + circuit.recordSuccess(); + return result; + } catch (err) { + lastError = err instanceof Error ? err : new Error(String(err)); + + if (!isNetworkFetchError(lastError.message)) { + // Non-network error (e.g. CUDA unavailable) β€” propagate without retry + throw lastError; + } + + circuit.recordFailure(); + + if (circuit.isOpen()) { + // Circuit just tripped β€” fail fast, no more retries + throw new Error( + `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${circuit.failureThreshold} consecutive failures`, + ); + } + + if (attempt < maxAttempts - 1) { + const delay = baseDelayMs * Math.pow(2, attempt); + onRetry?.(attempt + 1, maxAttempts, lastError); + await sleep(delay); + } + } + } + + // All retries exhausted β€” throw the last network error so isNetworkFetchError + // patterns in the calling code still match and surface HF_ENDPOINT guidance. + throw lastError; +} diff --git a/gitnexus/src/mcp/core/embedder.ts b/gitnexus/src/mcp/core/embedder.ts index ff3a12fd8..f529f2805 100644 --- a/gitnexus/src/mcp/core/embedder.ts +++ b/gitnexus/src/mcp/core/embedder.ts @@ -12,7 +12,11 @@ import { httpEmbedQuery, } from '../../core/embeddings/http-client.js'; import { resolveEmbeddingConfig } from '../../core/embeddings/config.js'; -import { applyHfEnvOverrides } from '../../core/embeddings/hf-env.js'; +import { + applyHfEnvOverrides, + isHfDownloadFailure, + withHfDownloadRetry, +} from '../../core/embeddings/hf-env.js'; import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js'; import { logger } from '../../core/logger.js'; @@ -69,23 +73,39 @@ export const initEmbedder = async (): Promise => { silenceStdout(); process.stderr.write = (() => true) as any; try { - embedderInstance = await (pipeline as any)('feature-extraction', MODEL_ID, { - device: device, - dtype: 'fp32', - session_options: { - logSeverityLevel: 3, - intraOpNumThreads: embeddingConfig.threads, - interOpNumThreads: 1, - executionMode: 'sequential', - }, - }); + embedderInstance = await withHfDownloadRetry(() => + pipeline('feature-extraction', MODEL_ID, { + device: device, + dtype: 'fp32', + session_options: { + logSeverityLevel: 3, + intraOpNumThreads: embeddingConfig.threads, + interOpNumThreads: 1, + executionMode: 'sequential', + }, + }), + ); } finally { restoreStdout(); process.stderr.write = realStderrWrite; } logger.info({ device }, 'GitNexus: Embedding model loaded'); return embedderInstance!; - } catch { + } catch (deviceError) { + // Network errors and circuit-open errors are not device-specific β€” + // they will fail the same way on every device. Rethrow immediately + // with actionable HF_ENDPOINT guidance rather than silently falling + // back to the next device. + const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError); + if (isHfDownloadFailure(errMsg)) { + const endpointHint = process.env.HF_ENDPOINT + ? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.` + : `huggingface.co may be unreachable from your network.\n` + + ` Set HF_ENDPOINT to a mirror and retry:\n` + + ` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` + + ` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`; + throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`); + } if (device === 'cpu') throw new Error('Failed to load embedding model'); } } diff --git a/gitnexus/test/unit/hf-env.test.ts b/gitnexus/test/unit/hf-env.test.ts index 6a5697059..fa8f23bd3 100644 --- a/gitnexus/test/unit/hf-env.test.ts +++ b/gitnexus/test/unit/hf-env.test.ts @@ -1,7 +1,20 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import os from 'node:os'; import { join } from 'node:path'; -import { applyHfEnvOverrides, type HfEnvSubset } from '../../src/core/embeddings/hf-env.js'; +import { + applyHfEnvOverrides, + isNetworkFetchError, + isHfDownloadFailure, + isHfCircuitOpenError, + HfDownloadCircuitBreaker, + withDownloadTimeout, + withHfDownloadRetry, + CIRCUIT_OPEN_TAG, + HF_MAX_ATTEMPTS, + HF_MAX_TIMEOUT_MS, + HF_MAX_ATTEMPTS_CAP, + type HfEnvSubset, +} from '../../src/core/embeddings/hf-env.js'; describe('applyHfEnvOverrides', () => { let envStub: HfEnvSubset; @@ -82,3 +95,399 @@ describe('applyHfEnvOverrides', () => { expect(envStub.remoteHost).toBe('https://hf-mirror.com/'); }); }); + +describe('isNetworkFetchError', () => { + it('returns true for "fetch failed" (the undici error seen on macOS/Node 24)', () => { + expect(isNetworkFetchError('fetch failed')).toBe(true); + }); + + it('returns true for ECONNREFUSED', () => { + expect(isNetworkFetchError('connect ECONNREFUSED 13.45.67.89:443')).toBe(true); + }); + + it('returns true for ENOTFOUND (DNS failure)', () => { + expect(isNetworkFetchError('getaddrinfo ENOTFOUND huggingface.co')).toBe(true); + }); + + it('returns true for ETIMEDOUT', () => { + expect(isNetworkFetchError('connect ETIMEDOUT 13.45.67.89:443')).toBe(true); + }); + + it('returns true for ECONNRESET', () => { + expect(isNetworkFetchError('read ECONNRESET')).toBe(true); + }); + + it('returns false for generic model-load errors (ONNX device failure)', () => { + expect(isNetworkFetchError('Failed to initialize CUDA backend')).toBe(false); + }); + + it('returns false for empty string', () => { + expect(isNetworkFetchError('')).toBe(false); + }); + + it('returns false for module-not-found errors', () => { + expect(isNetworkFetchError('Cannot find module onnxruntime-node')).toBe(false); + }); +}); + +describe('isHfCircuitOpenError', () => { + it('returns true for a circuit-open tag message', () => { + expect(isHfCircuitOpenError(`${CIRCUIT_OPEN_TAG}: circuit is open`)).toBe(true); + }); + + it('returns false for a plain network error', () => { + expect(isHfCircuitOpenError('fetch failed')).toBe(false); + }); +}); + +describe('isHfDownloadFailure', () => { + it('returns true for network fetch errors', () => { + expect(isHfDownloadFailure('ECONNREFUSED 127.0.0.1:443')).toBe(true); + }); + + it('returns true for circuit-open errors', () => { + expect(isHfDownloadFailure(`${CIRCUIT_OPEN_TAG}: open`)).toBe(true); + }); + + it('returns false for ONNX device errors', () => { + expect(isHfDownloadFailure('Failed to initialize CUDA')).toBe(false); + }); +}); + +describe('HfDownloadCircuitBreaker', () => { + it('starts in closed state', () => { + const cb = new HfDownloadCircuitBreaker(); + expect(cb.isOpen()).toBe(false); + expect(cb.state).toBe('closed'); + }); + + it('opens after reaching the failure threshold', () => { + const cb = new HfDownloadCircuitBreaker(3); + cb.recordFailure(); + cb.recordFailure(); + expect(cb.isOpen()).toBe(false); + cb.recordFailure(); // threshold reached + expect(cb.isOpen()).toBe(true); + expect(cb.state).toBe('open'); + }); + + it('closes on recordSuccess after being open', () => { + const cb = new HfDownloadCircuitBreaker(1); + cb.recordFailure(); + expect(cb.isOpen()).toBe(true); + cb.recordSuccess(); + expect(cb.isOpen()).toBe(false); + expect(cb.state).toBe('closed'); + }); + + it('transitions to half-open after the reset timeout', () => { + vi.useFakeTimers(); + try { + const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); + cb.recordFailure(); + expect(cb.isOpen()).toBe(true); + vi.advanceTimersByTime(200); + expect(cb.isOpen()).toBe(false); + expect(cb.state).toBe('half-open'); + } finally { + vi.useRealTimers(); + } + }); + + it('reset() restores closed state', () => { + const cb = new HfDownloadCircuitBreaker(1); + cb.recordFailure(); + expect(cb.isOpen()).toBe(true); + cb.reset(); + expect(cb.isOpen()).toBe(false); + expect(cb.state).toBe('closed'); + }); + + it('re-opens when a failure is recorded in half-open state', () => { + vi.useFakeTimers(); + try { + const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); + cb.recordFailure(); // opens the circuit + vi.advanceTimersByTime(200); // advance past reset timeout + expect(cb.state).toBe('half-open'); // getter transitions _state to half-open + cb.recordFailure(); // failure in half-open β†’ re-opens + expect(cb.isOpen()).toBe(true); + expect(cb.state).toBe('open'); + } finally { + vi.useRealTimers(); + } + }); + + it('closes the circuit when success is recorded in half-open state', () => { + vi.useFakeTimers(); + try { + const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); + cb.recordFailure(); // opens the circuit + vi.advanceTimersByTime(200); // advance past reset timeout + expect(cb.state).toBe('half-open'); + cb.recordSuccess(); // success in half-open β†’ closes + expect(cb.isOpen()).toBe(false); + expect(cb.state).toBe('closed'); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe('withDownloadTimeout', () => { + it('resolves when fn completes before the timeout', async () => { + const result = await withDownloadTimeout(() => Promise.resolve(42), 1_000); + expect(result).toBe(42); + }); + + it('rejects with ETIMEDOUT when fn takes too long', async () => { + vi.useFakeTimers(); + try { + const neverResolves = () => new Promise(() => {}); + const promise = withDownloadTimeout(neverResolves, 20); + vi.advanceTimersByTime(30); + await expect(promise).rejects.toThrow('ETIMEDOUT'); + } finally { + vi.useRealTimers(); + } + }); + + it('propagates non-timeout errors from fn', async () => { + await expect( + withDownloadTimeout(() => Promise.reject(new Error('download error')), 1_000), + ).rejects.toThrow('download error'); + }); +}); + +describe('withHfDownloadRetry', () => { + it('returns the result on first success', async () => { + const fn = vi.fn().mockResolvedValue('ok'); + const cb = new HfDownloadCircuitBreaker(); + const result = await withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 }); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('retries on network errors and succeeds on second attempt', async () => { + const fn = vi.fn().mockRejectedValueOnce(new Error('fetch failed')).mockResolvedValue('ok'); + const cb = new HfDownloadCircuitBreaker(); + const result = await withHfDownloadRetry(fn, { + circuit: cb, + maxAttempts: 3, + baseDelayMs: 0, + }); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('throws the last network error after all attempts are exhausted', async () => { + const fn = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:443')); + const cb = new HfDownloadCircuitBreaker(99 /* high threshold */); + await expect( + withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0 }), + ).rejects.toThrow('ECONNREFUSED'); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it('does not retry non-network errors', async () => { + const fn = vi.fn().mockRejectedValue(new Error('Failed to initialize CUDA backend')); + const cb = new HfDownloadCircuitBreaker(); + await expect( + withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0 }), + ).rejects.toThrow('Failed to initialize CUDA backend'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('fails immediately when the circuit is already open', async () => { + const fn = vi.fn().mockResolvedValue('ok'); + const cb = new HfDownloadCircuitBreaker(1); + cb.recordFailure(); // open the circuit + await expect(withHfDownloadRetry(fn, { circuit: cb })).rejects.toThrow(CIRCUIT_OPEN_TAG); + expect(fn).not.toHaveBeenCalled(); + }); + + it('opens the circuit after failureThreshold failures and throws a circuit-open error', async () => { + const fn = vi.fn().mockRejectedValue(new Error('ENOTFOUND huggingface.co')); + const cb = new HfDownloadCircuitBreaker(2 /* threshold */, 60_000); + // First call: 2 attempts, threshold=2 β†’ circuit opens on 2nd failure + await expect( + withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 2, baseDelayMs: 0 }), + ).rejects.toThrow(CIRCUIT_OPEN_TAG); + expect(cb.isOpen()).toBe(true); + }); + + it('calls onRetry with correct arguments on each retry', async () => { + const fn = vi + .fn() + .mockRejectedValueOnce(new Error('fetch failed')) + .mockRejectedValueOnce(new Error('fetch failed')) + .mockResolvedValue('ok'); + const cb = new HfDownloadCircuitBreaker(99); + const onRetry = vi.fn(); + await withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0, onRetry }); + expect(onRetry).toHaveBeenCalledTimes(2); + expect(onRetry).toHaveBeenNthCalledWith( + 1, + 1, + 3, + expect.objectContaining({ message: 'fetch failed' }), + ); + expect(onRetry).toHaveBeenNthCalledWith( + 2, + 2, + 3, + expect.objectContaining({ message: 'fetch failed' }), + ); + }); + + it('resets the circuit on success', async () => { + const fn = vi.fn().mockResolvedValue('value'); + const cb = new HfDownloadCircuitBreaker(5); + cb.recordFailure(); + cb.recordFailure(); // 2 failures, circuit still closed + await withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 }); + expect(cb.state).toBe('closed'); + }); +}); + +describe('withHfDownloadRetry env overrides', () => { + let originalTimeout: string | undefined; + let originalMaxAttempts: string | undefined; + + beforeEach(() => { + originalTimeout = process.env.HF_DOWNLOAD_TIMEOUT_MS; + originalMaxAttempts = process.env.HF_MAX_ATTEMPTS; + delete process.env.HF_DOWNLOAD_TIMEOUT_MS; + delete process.env.HF_MAX_ATTEMPTS; + }); + + afterEach(() => { + if (originalTimeout === undefined) delete process.env.HF_DOWNLOAD_TIMEOUT_MS; + else process.env.HF_DOWNLOAD_TIMEOUT_MS = originalTimeout; + if (originalMaxAttempts === undefined) delete process.env.HF_MAX_ATTEMPTS; + else process.env.HF_MAX_ATTEMPTS = originalMaxAttempts; + }); + + it('HF_MAX_ATTEMPTS=1 gives exactly 1 attempt', async () => { + process.env.HF_MAX_ATTEMPTS = '1'; + const fn = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:443')); + const cb = new HfDownloadCircuitBreaker(99_999 /* high threshold */); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'ECONNREFUSED', + ); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('HF_MAX_ATTEMPTS=2 gives exactly 2 attempts', async () => { + process.env.HF_MAX_ATTEMPTS = '2'; + const fn = vi.fn().mockRejectedValue(new Error('ENOTFOUND huggingface.co')); + const cb = new HfDownloadCircuitBreaker(99_999); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'ENOTFOUND', + ); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('HF_MAX_ATTEMPTS=abc falls back to the built-in default', async () => { + process.env.HF_MAX_ATTEMPTS = 'abc'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new HfDownloadCircuitBreaker(99_999); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'fetch failed', + ); + expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS); + }); + + it('HF_MAX_ATTEMPTS=0 falls back to the built-in default', async () => { + process.env.HF_MAX_ATTEMPTS = '0'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new HfDownloadCircuitBreaker(99_999); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'fetch failed', + ); + expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS); + }); + + it('HF_MAX_ATTEMPTS=-1 falls back to the built-in default', async () => { + process.env.HF_MAX_ATTEMPTS = '-1'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new HfDownloadCircuitBreaker(99_999); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'fetch failed', + ); + expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS); + }); + + it('HF_MAX_ATTEMPTS is clamped to HF_MAX_ATTEMPTS_CAP', async () => { + process.env.HF_MAX_ATTEMPTS = '9999'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new HfDownloadCircuitBreaker(99_999 /* very high threshold */); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'fetch failed', + ); + expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS_CAP); + }); + + it('HF_MAX_ATTEMPTS=2.9 is floored to 2', async () => { + process.env.HF_MAX_ATTEMPTS = '2.9'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new HfDownloadCircuitBreaker(99_999); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'fetch failed', + ); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('HF_DOWNLOAD_TIMEOUT_MS is used as the per-attempt timeout when valid', async () => { + vi.useFakeTimers(); + try { + process.env.HF_DOWNLOAD_TIMEOUT_MS = '50'; + const neverResolves = () => new Promise(() => {}); + const cb = new HfDownloadCircuitBreaker(99); + const promise = withHfDownloadRetry(neverResolves, { circuit: cb, maxAttempts: 1 }); + vi.advanceTimersByTime(100); + await expect(promise).rejects.toThrow('ETIMEDOUT'); + } finally { + vi.useRealTimers(); + } + }); + + it('HF_DOWNLOAD_TIMEOUT_MS=-1 falls back to the built-in default', async () => { + process.env.HF_DOWNLOAD_TIMEOUT_MS = '-1'; + // Passing explicit timeoutMs=0 (no real wait) so the test doesn't block; + // we just verify that the env var rejection causes options.timeoutMs to be + // the default constant (not -1) by confirming the resolved value is used. + const fn = vi.fn().mockResolvedValue('ok'); + const cb = new HfDownloadCircuitBreaker(99); + // Provide explicit timeoutMs to avoid the default 5-minute wait + const result = await withHfDownloadRetry(fn, { circuit: cb, timeoutMs: 100 }); + expect(result).toBe('ok'); + }); + + it('HF_DOWNLOAD_TIMEOUT_MS is clamped to HF_MAX_TIMEOUT_MS', async () => { + vi.useFakeTimers(); + try { + // Set an env value exceeding the 30-minute cap + process.env.HF_DOWNLOAD_TIMEOUT_MS = String(HF_MAX_TIMEOUT_MS + 60_000); + const neverResolves = () => new Promise(() => {}); + const cb = new HfDownloadCircuitBreaker(99); + const promise = withHfDownloadRetry(neverResolves, { circuit: cb, maxAttempts: 1 }); + // Advance just past the 30-minute cap + vi.advanceTimersByTime(HF_MAX_TIMEOUT_MS + 1); + await expect(promise).rejects.toThrow('ETIMEDOUT'); + } finally { + vi.useRealTimers(); + } + }); + + it('explicit options override env vars', async () => { + process.env.HF_MAX_ATTEMPTS = '5'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new HfDownloadCircuitBreaker(99); + // explicit maxAttempts: 2 must win over HF_MAX_ATTEMPTS=5 + await expect( + withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 2, baseDelayMs: 0 }), + ).rejects.toThrow('fetch failed'); + expect(fn).toHaveBeenCalledTimes(2); + }); +});