diff --git a/gitnexus-web/e2e/folder-upload.spec.ts b/gitnexus-web/e2e/folder-upload.spec.ts new file mode 100644 index 000000000..ba31b752c --- /dev/null +++ b/gitnexus-web/e2e/folder-upload.spec.ts @@ -0,0 +1,112 @@ +import { test, expect } from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * E2E for the browser folder-upload flow (replaces the removed server-side + * directory picker). Mocks the backend so no live gitnexus server is needed. + */ + +const BACKEND_URL = 'http://localhost:4747'; + +let fixtureDir: string; + +test.beforeAll(() => { + // A tiny "repo" folder; Playwright sets webkitRelativePath = /. + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-upload-e2e-')); + fixtureDir = path.join(root, 'myrepo'); + fs.mkdirSync(path.join(fixtureDir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(fixtureDir, 'README.md'), '# hi\n'); + fs.writeFileSync(path.join(fixtureDir, 'src', 'index.ts'), 'export const x = 1;\n'); +}); + +test.beforeEach(async ({ page }) => { + await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] })); + await page.route(`${BACKEND_URL}/api/info`, (route) => + route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }), + ); + await page.route(`${BACKEND_URL}/api/heartbeat`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: ':ok\n\n', + }), + ); +}); + +test('uploading a folder posts a multipart upload and starts analysis', async ({ page }) => { + let uploadContentType = ''; + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + uploadContentType = route.request().headers()['content-type'] ?? ''; + await route.fulfill({ json: { jobId: 'job-e2e', status: 'analyzing' } }); + }); + // SSE progress → immediately complete. + await page.route(`${BACKEND_URL}/api/analyze/job-e2e/progress`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }), + ); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); + + // Select the fixture folder via the hidden webkitdirectory input. + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + + // The upload endpoint should be hit with a multipart body, and the UI should + // leave the input phase (upload button no longer shown). + await expect.poll(() => uploadContentType).toContain('multipart/form-data'); + await expect(page.locator('[data-testid="upload-folder"]')).toBeHidden({ timeout: 10_000 }); +}); + +test('switching modes mid-upload aborts it and never shows progress', async ({ page }) => { + // Hold the upload response until the test releases it, so the mode switch + // happens while the POST is in flight (the review 4470339833 repro). + let releaseUpload!: () => void; + const uploadGate = new Promise((res) => (releaseUpload = res)); + let uploadAborted = false; + let progressOpened = false; + + // The client-side AbortController kills the POST at mode-switch time; that + // surfaces as a failed request (net::ERR_ABORTED), not as a response. + page.on('requestfailed', (req) => { + if (req.url().includes('/api/analyze/upload') && /ABORTED/.test(req.failure()?.errorText ?? '')) + uploadAborted = true; + }); + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + await uploadGate; + await route.fulfill({ json: { jobId: 'job-stale', status: 'analyzing' } }).catch(() => {}); // the request may already be gone — that's the point + }); + await page.route(`${BACKEND_URL}/api/analyze/job-stale/progress`, (route) => { + progressOpened = true; + return route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }); + }); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + await expect(page.locator('[data-testid="upload-progress"]')).toBeVisible(); + + // Switch back to GitHub while the upload POST is still pending, then let + // the (now-stale) route handler finish. + await page.getByRole('tab', { name: 'GitHub URL' }).click(); + await expect.poll(() => uploadAborted, { timeout: 10_000 }).toBe(true); + releaseUpload(); + + // The GitHub form stays clean (no error, immediately usable), and no SSE + // progress stream is ever opened by the stale upload. + await expect(page.getByPlaceholder('https://github.com/owner/repo')).toBeEditable(); + await expect(page.locator('[data-testid="upload-progress"]')).toBeHidden(); + expect(progressOpened).toBe(false); +}); diff --git a/gitnexus-web/e2e/onboarding.spec.ts b/gitnexus-web/e2e/onboarding.spec.ts index 5b70899c6..147f55632 100644 --- a/gitnexus-web/e2e/onboarding.spec.ts +++ b/gitnexus-web/e2e/onboarding.spec.ts @@ -218,8 +218,8 @@ test.describe('Flow 3: Analyze form', () => { // Switch to Local Folder tab await page.getByRole('tab', { name: 'Local Folder' }).click(); - // Browse button should be visible - await expect(page.getByText('Browse for folder')).toBeVisible(); + // Upload-a-folder button should be visible (browser folder upload) + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); await page.screenshot({ path: testInfo.outputPath('local-folder-tab.png') }); }); diff --git a/gitnexus-web/src/components/RepoAnalyzer.tsx b/gitnexus-web/src/components/RepoAnalyzer.tsx index 0b7f0abbd..613284830 100644 --- a/gitnexus-web/src/components/RepoAnalyzer.tsx +++ b/gitnexus-web/src/components/RepoAnalyzer.tsx @@ -21,9 +21,11 @@ import { startAnalyze, cancelAnalyze, streamAnalyzeProgress, + uploadFolder, type JobProgress, } from '../services/backend-client'; import { AnalyzeProgress } from './AnalyzeProgress'; +import { filterRepoFiles } from '@/lib/upload-filter'; import { useTranslation } from 'react-i18next'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -165,8 +167,11 @@ export interface RepoAnalyzerProps { export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProps) => { const { t } = useTranslation(['common', 'errors', 'onboarding']); const inputId = useId(); - const folderInputRef = useRef(null); const [mode, setMode] = useState('github'); + const [uploading, setUploading] = useState(false); + const [uploadSummary, setUploadSummary] = useState<{ count: number; dropped: number } | null>( + null, + ); const [githubUrl, setGithubUrl] = useState(''); const [gitlabUrl, setGitlabUrl] = useState(''); const [localPath, setLocalPath] = useState(''); @@ -181,28 +186,73 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const jobIdRef = useRef(null); const sseControllerRef = useRef(null); + // Owns the in-flight analyze/upload request. The controller doubles as the + // staleness token: each request captures its own controller in a closure and + // bails after the await when that controller was aborted, so a resolution + // arriving after a mode switch / cancel / unmount can never drive state. + const requestControllerRef = useRef(null); const completeTimerRef = useRef | null>(null); + const folderInputRef = useRef(null); useEffect(() => { return () => { sseControllerRef.current?.abort(); + requestControllerRef.current?.abort(); if (completeTimerRef.current) clearTimeout(completeTimerRef.current); }; }, []); + // Abort any in-flight analyze/upload request so its settlement can't drive + // state. Aborting is load-bearing: once a mode switch resets `uploading`, + // the `uploading || isLoading` re-entry guard no longer covers the stale + // request — only its aborted signal does. + const invalidateRequest = (): void => { + requestControllerRef.current?.abort(); + requestControllerRef.current = null; + }; + + // Invalidate the previous request and hand the caller a fresh controller. + const renewRequestController = (): AbortController => { + invalidateRequest(); + const controller = new AbortController(); + requestControllerRef.current = controller; + return controller; + }; + + // An upload that resolved after invalidation has still created a server-side + // job; cancel it so the single analyze slot isn't held for the job's full + // duration. Upload-path only: every upload owns a fresh job (the server + // stages each upload into a unique dir, never dedup-aliasing), whereas URL + // analyzes dedup-alias by repo — the returned jobId may belong to a job + // another session (or this user's own resubmit) is actively watching, so + // cancelling on that path could kill a live analysis. A stale URL job is + // left to finish: a same-URL resubmit re-attaches to it via dedup, and the + // server's job timeout/TTL sweep bounds the slot occupancy. + const cancelStaleUploadJob = (jobId: string): void => { + void cancelAnalyze(jobId).catch(() => {}); + }; + const handleModeChange = (m: InputMode) => { + // ModeTabs fires onChange on every click, including the already-active + // tab — never abort the user's own in-flight request for a no-op click. + if (m === mode) return; + invalidateRequest(); setMode(m); setGithubUrl(''); setGitlabUrl(''); setLocalPath(''); setValidationError(null); + setUploadSummary(null); + setUploading(false); + // An aborted request no longer resolves to move `phase` off 'starting'; + // reset so the new mode's form is immediately usable (also clears a stale + // 'error' phase). Only reachable while showInput is true. + setPhase('input'); }; - // Use the browser's native directory picker (webkitdirectory doesn't give paths, - // so we use a text input + a "Browse" button that opens a standard file input - // to let users pick files from the folder — the path is typed manually since - // browsers don't expose absolute paths for security reasons). - // For local paths, the user types or pastes the absolute path. + // Local-folder mode uploads the selected folder's files (the browser never + // exposes an absolute path, so the old typed-path/browse approach couldn't + // work — see handleFolderUpload). A typed server path is also still accepted. const canSubmit = mode === 'github' @@ -228,6 +278,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp setValidationError(null); setPhase('starting'); + // Staleness guard only (no wire abort): the POST is short-lived and + // self-terminates, but its resolution must not drive state after a mode + // switch / cancel / unmount invalidated this request. + const controller = renewRequestController(); try { const request = mode === 'github' @@ -236,8 +290,9 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp ? { url: gitlabUrl.trim() } : { path: localPath.trim() }; const { jobId } = await startAnalyze(request); - jobIdRef.current = jobId; - setPhase('analyzing'); + // Stale resolution: return without cancelling — URL jobIds may be + // dedup-aliased to a job another session owns (see cancelStaleUploadJob). + if (controller.signal.aborted) return; const nameSource = mode === 'github' @@ -245,29 +300,84 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp : mode === 'gitlab' ? gitlabUrl.trim() : localPath.trim(); - const controller = streamAnalyzeProgress( - jobId, - (p) => setProgress(p), - (data) => { - const name = - data.repoName ?? - nameSource.split(/[/\\]/).filter(Boolean).at(-1) ?? - t('onboarding:repoAnalyzer.defaultRepoName'); - setCompletedRepoName(name); - setPhase('done'); - sseControllerRef.current = null; - completeTimerRef.current = setTimeout(() => { - completeTimerRef.current = null; - onComplete(name); - }, 1200); - }, - (errMsg) => { - setValidationError(errMsg || t('errors:analysisFailed')); - setPhase('error'); - }, - ); - sseControllerRef.current = controller; + trackJob(jobId, nameSource); } catch (err) { + // Unmount aborts the controller, so this also covers the unmounted case. + if (controller.signal.aborted) return; + setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); + setPhase('error'); + } + }; + + // Drive an already-created analysis job through the SSE progress stream to + // completion. Shared by the path/URL analyze flow and the folder-upload flow. + const trackJob = (jobId: string, fallbackNameSource: string | null) => { + // Callers reach here only with a live (non-aborted) request controller, so + // the component is mounted — unmount aborts the controller. + jobIdRef.current = jobId; + setPhase('analyzing'); + const controller = streamAnalyzeProgress( + jobId, + (p) => setProgress(p), + (data) => { + const name = + data.repoName ?? + (fallbackNameSource + ? fallbackNameSource.split(/[/\\]/).filter(Boolean).at(-1) + : undefined) ?? + t('onboarding:repoAnalyzer.defaultRepoName'); + setCompletedRepoName(name); + setPhase('done'); + sseControllerRef.current = null; + completeTimerRef.current = setTimeout(() => { + completeTimerRef.current = null; + onComplete(name); + }, 1200); + }, + (errMsg) => { + setValidationError(errMsg || t('errors:analysisFailed')); + setPhase('error'); + }, + ); + sseControllerRef.current = controller; + }; + + // Upload a browser-selected folder (webkitdirectory) and start analysis. The + // upload endpoint returns a jobId, which then joins the normal SSE flow. + const handleFolderUpload = async (fileList: FileList) => { + if (uploading || isLoading) return; // guard against a concurrent upload + const { files, manifest, droppedCount } = filterRepoFiles(fileList); + if (files.length === 0) { + setValidationError(t('onboarding:repoAnalyzer.upload.empty')); + return; + } + setValidationError(null); + setUploadSummary({ count: files.length, dropped: droppedCount }); + setUploading(true); + setPhase('starting'); + // The selected folder's name (manifest entries are `/`) is a + // sensible fallback if the server's complete event omits repoName. + const folderName = manifest[0]?.split('/')[0] ?? null; + const controller = renewRequestController(); + try { + const { jobId } = await uploadFolder(files, manifest, controller.signal); + if (controller.signal.aborted) { + // The abort raced the response: the server already created the job. + // (Unmount aborts the controller, so this also covers unmounted.) + cancelStaleUploadJob(jobId); + return; + } + setUploading(false); + trackJob(jobId, folderName); + } catch (err) { + // An abort surfaces in two shapes — BackendError('Request aborted') + // when it lands during fetch, raw AbortError when it lands during the + // response-body read — so branch on the closure controller's signal, + // never on the error identity. In the second shape the server may have + // already launched a job whose id we never learn; that orphan is bounded + // by the server's job timeout and terminal-job TTL sweep. + if (controller.signal.aborted) return; + setUploading(false); setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); setPhase('error'); } @@ -276,6 +386,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const handleCancel = async () => { sseControllerRef.current?.abort(); sseControllerRef.current = null; + // Defensive: no UI path can reach handleCancel while a request is in + // flight (the cancel affordance renders only at phase === 'analyzing'), + // but invalidate it anyway so the guard topology has no holes. + invalidateRequest(); if (jobIdRef.current) { try { await cancelAnalyze(jobIdRef.current); @@ -284,6 +398,8 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp } setPhase('input'); setProgress({ phase: 'queued', percent: 0, message: t('common:analyzePhases.queued') }); + setUploading(false); + setUploadSummary(null); }; const isLoading = phase === 'starting'; @@ -443,35 +559,51 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp )} - {/* Native folder picker + Browse button — below the input */} + {/* Upload a folder from your computer — no server path or mount needed. + The browser can't expose an absolute path, so we upload the files. */} { - const files = e.target.files; - if (files && files.length > 0) { - const rel = files[0].webkitRelativePath; - const folderName = rel.split('/')[0]; - if (folderName) { - setLocalPath(folderName); - setValidationError(null); - } + if (e.target.files && e.target.files.length > 0) { + handleFolderUpload(e.target.files); } e.target.value = ''; }} /> + {uploading && ( +
+
+
+
+

+ {t('onboarding:repoAnalyzer.upload.uploading')} +

+
+ )} + {uploadSummary && !uploading && phase !== 'error' && ( +

+ {t('onboarding:repoAnalyzer.upload.selected', { + fileCount: uploadSummary.count, + dropped: uploadSummary.dropped, + })} +

+ )}
)} diff --git a/gitnexus-web/src/lib/upload-filter.test.ts b/gitnexus-web/src/lib/upload-filter.test.ts new file mode 100644 index 000000000..e97b9ec2c --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import { filterRepoFiles, MAX_FILE_BYTES } from './upload-filter'; + +type FileLike = { name: string; size: number; webkitRelativePath?: string }; + +function f(webkitRelativePath: string, size = 10): FileLike { + const name = webkitRelativePath.split('/').pop() ?? webkitRelativePath; + return { name, size, webkitRelativePath }; +} + +describe('filterRepoFiles', () => { + it('keeps source files and builds an order-aligned manifest', () => { + const input = [f('repo/src/index.ts', 100), f('repo/README.md', 50)]; + const r = filterRepoFiles(input); + expect(r.files).toHaveLength(2); + expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/README.md']); + expect(r.totalBytes).toBe(150); + expect(r.droppedCount).toBe(0); + }); + + it('excludes .git / node_modules / build dirs anywhere in the path', () => { + const input = [ + f('repo/.git/HEAD'), + f('repo/node_modules/x/index.js'), + f('repo/dist/bundle.js'), + f('repo/src/app.ts'), + f('repo/.gitnexus/meta.json'), + ]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/src/app.ts']); + expect(r.droppedCount).toBe(4); + }); + + it('drops files over the per-file size cap', () => { + const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/small.ts']); + expect(r.droppedCount).toBe(1); + }); + + it('falls back to name when webkitRelativePath is absent', () => { + const r = filterRepoFiles([{ name: 'lone.ts', size: 5 }]); + expect(r.manifest).toEqual(['lone.ts']); + }); +}); diff --git a/gitnexus-web/src/lib/upload-filter.ts b/gitnexus-web/src/lib/upload-filter.ts new file mode 100644 index 000000000..a24520f74 --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.ts @@ -0,0 +1,73 @@ +/** + * Client-side pre-filter for a webkitdirectory folder upload. + * + * Drops VCS metadata, dependency/build directories, and oversized files before + * upload — `.git` alone is often larger than the working tree — so payloads + * stay small and the upload matches what the analyzer actually needs. Produces + * an order-aligned `manifest` of webkitRelativePaths (the server keys on this, + * not the multipart filename, which browsers rewrite). + */ + +/** Directory names excluded anywhere in a file's path. */ +export const EXCLUDED_DIRS = new Set([ + '.git', + '.hg', + '.svn', + 'node_modules', + 'vendor', + '.venv', + '__pycache__', + 'target', + 'dist', + 'build', + 'out', + '.next', + '.nuxt', + '.cache', + 'coverage', + '.idea', + '.gitnexus', +]); + +/** Per-file size cap; matches the server's per-file limit. */ +export const MAX_FILE_BYTES = 25 * 1024 * 1024; + +export interface FilterResult { + files: File[]; + manifest: string[]; + droppedCount: number; + totalBytes: number; +} + +type FileLike = Pick & { webkitRelativePath?: string }; + +/** + * Filter a webkitdirectory `FileList` (or array) into the files to upload plus + * their relative-path manifest. + */ +export function filterRepoFiles(input: ArrayLike): FilterResult { + const files: File[] = []; + const manifest: string[] = []; + let droppedCount = 0; + let totalBytes = 0; + + for (let i = 0; i < input.length; i++) { + const f = input[i]; + const rel = + f.webkitRelativePath && f.webkitRelativePath.length > 0 ? f.webkitRelativePath : f.name; + const segments = rel.split('/'); + if (segments.some((s) => EXCLUDED_DIRS.has(s))) { + droppedCount++; + continue; + } + if (f.size > MAX_FILE_BYTES) { + droppedCount++; + continue; + } + files.push(f as File); + manifest.push(rel); + totalBytes += f.size; + } + + return { files, manifest, droppedCount, totalBytes }; +} diff --git a/gitnexus-web/src/locales/en/onboarding.json b/gitnexus-web/src/locales/en/onboarding.json index cd12c2095..8be58efce 100644 --- a/gitnexus-web/src/locales/en/onboarding.json +++ b/gitnexus-web/src/locales/en/onboarding.json @@ -61,7 +61,12 @@ "gitlabRepositoryUrl": "GitLab Repository URL", "gitlabSupported": "Supports GitLab.com and self-hosted GitLab instances.", "localFolderPath": "Local Folder Path", - "browseForFolder": "Browse for folder", - "hideBackground": "Hide (analysis continues in background)" + "hideBackground": "Hide (analysis continues in background)", + "upload": { + "button": "Upload a folder", + "uploading": "Uploading…", + "selected": "{{fileCount}} files ready ({{dropped}} skipped: .git, node_modules, build output)", + "empty": "No analyzable files found in that folder." + } } } diff --git a/gitnexus-web/src/locales/zh-CN/onboarding.json b/gitnexus-web/src/locales/zh-CN/onboarding.json index 6199511f3..f20e60b6d 100644 --- a/gitnexus-web/src/locales/zh-CN/onboarding.json +++ b/gitnexus-web/src/locales/zh-CN/onboarding.json @@ -61,7 +61,12 @@ "gitlabRepositoryUrl": "GitLab 仓库 URL", "gitlabSupported": "支持 GitLab.com 和自托管 GitLab 实例。", "localFolderPath": "本地文件夹路径", - "browseForFolder": "浏览文件夹", - "hideBackground": "隐藏（分析继续在后台进行）" + "hideBackground": "隐藏（分析继续在后台进行）", + "upload": { + "button": "上传文件夹", + "uploading": "上传中…", + "selected": "已准备 {{fileCount}} 个文件（已跳过 {{dropped}} 个：.git、node_modules、构建产物）", + "empty": "该文件夹中未找到可分析的文件。" + } } } diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index e887e3901..286a13187 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -283,12 +283,11 @@ const fetchWithTimeout = async ( ): Promise => { // Merge the external caller signal (if any) with an // `AbortSignal.timeout()` so a timer-fired abort produces a - // `DOMException` with `name === 'TimeoutError'` — which - // `resilientFetch` correctly classifies as terminal-network (no - // retry, no breaker hit). A manual `AbortController.abort()` would - // produce `name === 'AbortError'` and route through the - // retryable-network branch, which mis-penalizes the breaker for - // user-side network slowness. + // `DOMException` with `name === 'TimeoutError'`. Both shapes are + // breaker-safe: `resilientFetch` classifies TimeoutError AND a manual + // `AbortController.abort()`'s AbortError as terminal-network (no + // retry, breaker-neutral via recordNeutral), so caller-driven + // cancellation never penalizes the breaker. const timeoutSignal = AbortSignal.timeout(timeoutMs); const externalSignal = init.signal; const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal; @@ -755,6 +754,35 @@ export const fetchClusterDetail = async (repo: string, name: string): Promise`) and start analysis. + * Sends the file blobs plus a JSON `manifest` of their relative paths — the + * multipart filename can't carry the path (browsers strip separators), so the + * manifest is the source of truth. Routed through fetchWithTimeout (the shared, + * origin-validated request path) rather than a raw XHR; returns the analysis + * jobId, which the caller drives through the normal SSE flow. + */ +export const uploadFolder = async ( + files: File[], + manifest: string[], + signal?: AbortSignal, +): Promise<{ jobId: string; status: string }> => { + const form = new FormData(); + // Manifest MUST precede the file parts (the server enforces this). + form.append('manifest', JSON.stringify(manifest)); + for (const f of files) form.append('files', f); + + const response = await fetchWithTimeout( + `${_backendUrl}/api/analyze/upload`, + { method: 'POST', body: form, signal }, + 5 * 60_000, // up to 5 min for large repos + ); + await assertOk(response); + return response.json() as Promise<{ jobId: string; status: string }>; +}; + // ── Analyze API ──────────────────────────────────────────────────────────── /** Start a server-side analysis job. */ diff --git a/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx b/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx new file mode 100644 index 000000000..094641b93 --- /dev/null +++ b/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx @@ -0,0 +1,218 @@ +/** + * Stale-request guards in RepoAnalyzer (PR #1850 review 4470339833). + * + * An analyze request (folder upload or URL analyze) that is still in flight + * when the user switches modes, cancels, or unmounts must not drive state + * when it later settles: no SSE stream, no phase/error flip — and a + * stale-but-created server job gets a fire-and-forget cancel so the single + * analyze slot is freed. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { RepoAnalyzer } from '../../src/components/RepoAnalyzer'; +import { i18nReady } from '../../src/i18n'; +import { + cancelAnalyze, + startAnalyze, + streamAnalyzeProgress, + uploadFolder, +} from '../../src/services/backend-client'; + +vi.mock('../../src/services/backend-client', () => ({ + startAnalyze: vi.fn(), + cancelAnalyze: vi.fn(), + streamAnalyzeProgress: vi.fn(), + uploadFolder: vi.fn(), +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +const JOB = { jobId: 'job-1', status: 'queued' }; + +/** Gate uploadFolder on a deferred promise and expose the signal it received. */ +function mockUploadWith(d: { promise: Promise }) { + let captured: AbortSignal | undefined; + vi.mocked(uploadFolder).mockImplementation((_files, _manifest, signal) => { + captured = signal; + return d.promise; + }); + return { signal: () => captured }; +} + +/** Render, switch to Local Folder mode, and fire a folder selection. */ +function startUpload() { + const view = render(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + fireEvent.change(screen.getByTestId('folder-upload-input'), { + target: { files: [new File(['x'], 'a.ts')] }, + }); + return view; +} + +beforeEach(async () => { + await i18nReady; + vi.clearAllMocks(); + vi.mocked(cancelAnalyze).mockResolvedValue(undefined as never); + vi.mocked(streamAnalyzeProgress).mockImplementation(() => new AbortController()); +}); + +describe('folder upload', () => { + it('a mode switch mid-upload makes the resolution inert and cancels the job', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'GitHub URL' })); + + // The wire abort happened at mode-switch time, not at resolution time. + expect(upload.signal()?.aborted).toBe(true); + + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + expect(cancelAnalyze).toHaveBeenCalledWith('job-1'); + // The GitHub form is clean and submittable (phase back to 'input'). + expect(screen.getByRole('textbox')).toBeEnabled(); + expect(screen.queryByTestId('upload-progress')).not.toBeInTheDocument(); + }); + + it.each([ + ['BackendError shape', new Error('Request aborted')], + ['raw AbortError shape', new DOMException('The operation was aborted.', 'AbortError')], + ])('an aborted rejection is silent — %s', async (_label, err) => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'GitHub URL' })); + await act(async () => { + d.reject(err); + }); + + expect(screen.queryByText('Request aborted')).not.toBeInTheDocument(); + expect(screen.queryByText('The operation was aborted.')).not.toBeInTheDocument(); + expect(screen.getByRole('textbox')).toBeEnabled(); + }); + + it('a same-tab click does not abort the in-flight upload', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + + expect(upload.signal()?.aborted).toBe(false); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('an unmount mid-upload makes the resolution inert', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + const { unmount } = startUpload(); + unmount(); + + expect(upload.signal()?.aborted).toBe(true); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + }); + + it('the happy path still tracks the job', async () => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(vi.mocked(streamAnalyzeProgress).mock.calls[0][0]).toBe('job-1'); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('a genuine error still surfaces', async () => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + await act(async () => { + d.reject(new Error('upload exploded')); + }); + + expect(screen.getByText('upload exploded')).toBeInTheDocument(); + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + }); +}); + +describe('URL analyze', () => { + function startGithubAnalyze() { + render(); + fireEvent.change(screen.getByRole('textbox'), { + target: { value: 'https://github.com/owner/repo' }, + }); + fireEvent.click(screen.getByRole('button', { name: /Analyze Repository/ })); + } + + it('a mode switch mid-analyze makes the resolution inert without cancelling', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + await act(async () => { + d.resolve({ jobId: 'job-2', status: 'queued' }); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + // No cancel on the URL path: the jobId may be dedup-aliased to a job + // another session owns, so cancelling could kill a live analysis. + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('a stale rejection is silent', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + await act(async () => { + d.reject(new Error('analyze exploded')); + }); + + expect(screen.queryByText('analyze exploded')).not.toBeInTheDocument(); + expect(screen.getByTestId('upload-folder')).toBeEnabled(); + }); + + it('the happy path still tracks the job', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + await act(async () => { + d.resolve({ jobId: 'job-3', status: 'queued' }); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(vi.mocked(streamAnalyzeProgress).mock.calls[0][0]).toBe('job-3'); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 7358d3254..9fad8f440 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -14,6 +14,7 @@ "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -51,6 +52,7 @@ "gitnexus": "dist/cli/index.js" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", @@ -1703,6 +1705,16 @@ "@types/node": "*" } }, + "node_modules/@types/busboy": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz", + "integrity": "sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2213,6 +2225,17 @@ "node": "18 || 20 || >=22" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -4825,6 +4848,14 @@ "dev": true, "license": "MIT" }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 3543a339b..6fa0c9007 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -59,6 +59,7 @@ "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -93,6 +94,7 @@ "uuid": "^14.0.0" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", diff --git a/gitnexus/src/server/analyze-launch.ts b/gitnexus/src/server/analyze-launch.ts new file mode 100644 index 000000000..184c5a813 --- /dev/null +++ b/gitnexus/src/server/analyze-launch.ts @@ -0,0 +1,170 @@ +/** + * Shared analyze-worker launcher. + * + * Forks the analyze worker for an already-resolved repo directory and owns the + * lock + auto-retry + IPC machinery. Used by both the JSON `/api/analyze` route + * and the multipart `/api/analyze/upload` route. Dependency-injected (like + * createAnalyzeUploadHandler) so the seam is testable and api.ts stays smaller. + * + * NOTE: this module must live alongside analyze-worker.{ts,js} — the worker + * path is resolved relative to `import.meta.url`. + */ + +import path from 'path'; +import { fork } from 'child_process'; +import { fileURLToPath, pathToFileURL } from 'url'; +import { createRequire } from 'node:module'; +import { getStoragePath } from '../storage/repo-manager.js'; +import { logger } from '../core/logger.js'; +import type { JobManager } from './analyze-job.js'; +import type { WorkerMessage } from './analyze-worker.js'; + +const _require = createRequire(import.meta.url); + +export interface LaunchDeps { + jobManager: JobManager; + backend: { init: () => Promise }; + acquireRepoLock: (key: string) => string | null; + releaseRepoLock: (key: string) => void; +} + +export interface LaunchOptions { + force?: boolean; + embeddings?: boolean; + dropEmbeddings?: boolean; + registryName?: string; +} + +const MAX_WORKER_RETRIES = 2; + +export function createLaunchAnalysisWorker(deps: LaunchDeps) { + const { jobManager, backend, acquireRepoLock, releaseRepoLock } = deps; + + return function launchAnalysisWorker( + job: { id: string }, + targetPath: string, + opts: LaunchOptions, + ): void { + // Acquire shared repo lock (keyed on storagePath to match embed handler) + const analyzeLockKey = getStoragePath(targetPath); + const lockErr = acquireRepoLock(analyzeLockKey); + if (lockErr) { + jobManager.updateJob(job.id, { status: 'failed', error: lockErr }); + return; + } + + jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' }); + + // ── Worker fork with auto-retry ────────────────────────────── + const callerPath = fileURLToPath(import.meta.url); + const isDev = callerPath.endsWith('.ts'); + const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js'; + const workerPath = path.join(path.dirname(callerPath), workerFile); + const tsxHookArgs: string[] = isDev + ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] + : []; + + const forkWorker = () => { + const currentJob = jobManager.getJob(job.id); + if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed') return; + + const child = fork(workerPath, [], { + execArgv: [...tsxHookArgs, '--max-old-space-size=8192'], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }); + + // Capture stderr for crash diagnostics + let stderrChunks = ''; + child.stderr?.on('data', (chunk: Buffer) => { + stderrChunks += chunk.toString(); + if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096); + }); + + child.on('message', (msg: WorkerMessage) => { + if (msg.type === 'progress') { + jobManager.updateJob(job.id, { + status: 'analyzing', + progress: { phase: msg.phase, percent: msg.percent, message: msg.message }, + }); + } else if (msg.type === 'complete') { + releaseRepoLock(analyzeLockKey); + // Reinitialize backend BEFORE marking complete — ensures the new repo + // is queryable when the client receives the SSE complete event. + backend + .init() + .then(() => { + jobManager.updateJob(job.id, { status: 'complete', repoName: msg.result.repoName }); + }) + .catch((err) => { + logger.error({ err }, 'backend.init() failed after analyze:'); + jobManager.updateJob(job.id, { + status: 'failed', + error: 'Server failed to reload after analysis. Try again.', + }); + }); + } else if (msg.type === 'error') { + releaseRepoLock(analyzeLockKey); + jobManager.updateJob(job.id, { status: 'failed', error: msg.message }); + } + }); + + child.on('error', (err) => { + releaseRepoLock(analyzeLockKey); + jobManager.updateJob(job.id, { + status: 'failed', + error: `Worker process error: ${err.message}`, + }); + }); + + child.on('exit', (code) => { + const j = jobManager.getJob(job.id); + if (!j || j.status === 'complete' || j.status === 'failed') return; + + // Worker crashed — attempt retry if under the limit + if (j.retryCount < MAX_WORKER_RETRIES) { + j.retryCount++; + const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s + const lastErr = stderrChunks.trim().split('\n').pop() || ''; + logger.warn( + `Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` + + (lastErr ? `: ${lastErr}` : ''), + ); + jobManager.updateJob(job.id, { + status: 'analyzing', + progress: { + phase: 'retrying', + percent: j.progress.percent, + message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`, + }, + }); + stderrChunks = ''; + setTimeout(forkWorker, delay); + } else { + // Exhausted retries — permanent failure + releaseRepoLock(analyzeLockKey); + jobManager.updateJob(job.id, { + status: 'failed', + error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`, + }); + } + }); + + // Register child for cancellation + timeout tracking + jobManager.registerChild(job.id, child); + + // Send start command to child + child.send({ + type: 'start', + repoPath: targetPath, + options: { + force: !!opts.force, + embeddings: !!opts.embeddings, + dropEmbeddings: !!opts.dropEmbeddings, + ...(opts.registryName ? { registryName: opts.registryName } : {}), + }, + }); + }; + + forkWorker(); + }; +} diff --git a/gitnexus/src/server/analyze-upload.ts b/gitnexus/src/server/analyze-upload.ts new file mode 100644 index 000000000..adb8b861b --- /dev/null +++ b/gitnexus/src/server/analyze-upload.ts @@ -0,0 +1,155 @@ +/** + * POST /api/analyze/upload — analyze a browser folder upload. + * + * Securely ingests the multipart upload into a sandbox (upload-ingest.ts), + * promotes it to a persistent app-controlled directory, and analyzes it via + * the same job/worker machinery as a git clone — never returning a server + * path to the client. Factored as a dependency-injected handler so the job + * machinery (createJob + the worker launcher) can be mocked in unit tests. + */ + +import path from 'path'; +import fsp from 'fs/promises'; +import type { Request, Response } from 'express'; +import type { IncomingMessage } from 'http'; +import { ingestUpload } from './upload-ingest.js'; +import { UPLOAD_ROOT, getUploadDir, deriveUploadName } from './upload-paths.js'; +import { BadRequestError } from './validation.js'; +import type { AnalyzeJob } from './analyze-job.js'; + +/** Minimal job shape the handler needs (a subset of the real AnalyzeJob). */ +export type UploadJobRef = Pick; + +/** Cap on collision-suffix attempts when allocating an upload dir name. */ +const MAX_NAME_COLLISION_TRIES = 100; + +export interface AnalyzeUploadDeps { + /** Create (or throw on busy) an analysis job for the given upload dir. */ + createJob: (params: { repoPath: string }) => UploadJobRef; + /** Launch the analyze worker against an already-resolved repo directory. */ + launch: (job: UploadJobRef, targetPath: string, opts: { registryName: string }) => void; + /** + * Mark a created job failed. The job occupies the single analysis slot from + * createJob onward, so ANY error before launch must release it — otherwise a + * leaked non-terminal job wedges all future analyses until restart. + */ + failJob: (jobId: string, error: string) => void; + /** Injectable for tests (defaults to the real ingestUpload). */ + ingest?: typeof ingestUpload; +} + +/** + * Find an available upload directory name, appending `-2`, `-3`, … on + * collision with an existing upload. Bounded to avoid an unbounded scan. + */ +async function pickAvailableName(base: string): Promise { + for (let i = 0; i < MAX_NAME_COLLISION_TRIES; i++) { + const name = i === 0 ? base : `${base}-${i + 1}`; + let dir: string; + try { + dir = getUploadDir(name); + } catch { + continue; + } + try { + await fsp.access(dir); + // exists → try the next suffix + } catch { + return name; // ENOENT → available + } + } + throw new BadRequestError( + `Could not allocate an upload directory after ${MAX_NAME_COLLISION_TRIES} attempts`, + 409, + ); +} + +export function createAnalyzeUploadHandler(deps: AnalyzeUploadDeps) { + const ingest = deps.ingest ?? ingestUpload; + + return async function handleAnalyzeUploadRequest(req: Request, res: Response): Promise { + let stageRoot: string | undefined; + let promotedDir: string | undefined; + let createdJobId: string | undefined; + let launched = false; + try { + const result = await ingest(req as IncomingMessage); + stageRoot = result.stageRoot; + + const baseName = deriveUploadName(result.topLevelName); + if (!baseName) { + throw new BadRequestError('Uploaded folder has no usable name'); + } + + // webkitRelativePath prefixes every entry with the picked folder, so the + // real repo root is stageRoot/. Validate it is a directory + // BEFORE taking the single analysis slot — a malformed (non-folder) + // upload must not be able to occupy the slot. + const innerRoot = path.join(result.stageRoot, result.topLevelName); + let innerIsDir = false; + try { + innerIsDir = (await fsp.stat(innerRoot)).isDirectory(); + } catch { + innerIsDir = false; + } + if (!innerIsDir) { + throw new BadRequestError('Upload must be a folder'); + } + + const finalName = await pickAvailableName(baseName); + const finalDir = getUploadDir(finalName); + + // createJob occupies the single analysis slot (throws 'already in + // progress' → 409). From here on, ANY error before launch MUST release + // the slot via failJob in the catch, or the server wedges all analyses. + let job: UploadJobRef; + try { + job = deps.createJob({ repoPath: finalDir }); + } catch (err) { + const msg = err instanceof Error ? err.message : ''; + if (msg.includes('already in progress')) { + throw new BadRequestError(msg, 409); + } + throw err; + } + createdJobId = job.id; + + // Promote staging → persistent upload dir. Both live under UPLOAD_ROOT's + // filesystem, so this rename stays atomic (no EXDEV). + await fsp.mkdir(UPLOAD_ROOT, { recursive: true }); + await fsp.rename(innerRoot, finalDir); + promotedDir = finalDir; + const oldStage = stageRoot; + stageRoot = undefined; + await fsp.rm(oldStage, { recursive: true, force: true }).catch(() => {}); + + // Drop any crafted index the upload may have carried (a `.gitnexus` + // segment passes containment); the worker will build a fresh one. + await fsp + .rm(path.join(finalDir, '.gitnexus'), { recursive: true, force: true }) + .catch(() => {}); + + deps.launch(job, finalDir, { registryName: finalName }); + launched = true; + + res.status(202).json({ jobId: job.id, status: job.status }); + } catch (err) { + // Release the single analysis slot if a job was created but never + // launched — otherwise the leaked queued job blocks all future analyses. + if (createdJobId && !launched) { + deps.failJob(createdJobId, err instanceof Error ? err.message : 'Upload failed'); + } + if (stageRoot) { + await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => {}); + } + if (promotedDir && !launched) { + await fsp.rm(promotedDir, { recursive: true, force: true }).catch(() => {}); + } + if (err instanceof BadRequestError) { + res.status(err.status).json({ error: err.message }); + return; + } + res.status(500).json({ error: 'Upload failed' }); + } + }; +} diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index 1cacd1d2f..d14fb0d39 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -21,26 +21,27 @@ interface StartMessage { options: AnalyzeOptions; } -interface ProgressMessage { +export interface ProgressMessage { type: 'progress'; phase: string; percent: number; message: string; } -interface CompleteMessage { +export interface CompleteMessage { type: 'complete'; // JSON-safe projection (no `pipelineResult` / live KnowledgeGraph). This // channel is default-JSON child_process IPC — see analyze-worker-ipc.ts. result: AnalyzeResultIpc; } -interface ErrorMessage { +export interface ErrorMessage { type: 'error'; message: string; } -type WorkerMessage = ProgressMessage | CompleteMessage | ErrorMessage; +/** Child → parent IPC messages. Shared with the parent-side launcher. */ +export type WorkerMessage = ProgressMessage | CompleteMessage | ErrorMessage; function send(msg: WorkerMessage) { process.send?.(msg); diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 6ab9cf1c3..67ae63009 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -30,11 +30,15 @@ import { searchFTSFromLbug } from '../core/search/bm25-index.js'; import { hybridSearch } from '../core/search/hybrid-search.js'; import { LocalBackend } from '../mcp/local/local-backend.js'; import { mountMCPEndpoints } from './mcp-http.js'; -import { fork } from 'child_process'; -import { fileURLToPath, pathToFileURL } from 'url'; +import { fileURLToPath } from 'url'; import { JobManager } from './analyze-job.js'; import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js'; import { extractRepoName, getCloneDir, cloneOrPull } from './git-clone.js'; +import { createAnalyzeUploadHandler } from './analyze-upload.js'; +import { requireLocalhostOrigin } from './middleware.js'; +import { createLaunchAnalysisWorker } from './analyze-launch.js'; +import { UPLOAD_ROOT } from './upload-paths.js'; +import { sweepStaleUploads } from './upload-sweep.js'; import { logger, flushLoggerSync } from '../core/logger.js'; const _require = createRequire(import.meta.url); @@ -741,6 +745,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const cleanupMcp = mountMCPEndpoints(app, backend); const jobManager = new JobManager(); + // Backstop: remove any upload staging dirs orphaned by a previous crash. + void sweepStaleUploads().catch(() => {}); + // Shared repo lock — prevents concurrent analyze + embed on the same repo path, // which would corrupt LadybugDB (analyze calls closeLbug + initLbug while embed has queries in flight). const activeRepoPaths = new Set(); @@ -757,6 +764,15 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => activeRepoPaths.delete(repoPath); }; + // Launch the analyze worker for an already-resolved repo directory. Shared by + // the JSON /api/analyze route and the multipart /api/analyze/upload route. + const launchAnalysisWorker = createLaunchAnalysisWorker({ + jobManager, + backend, + acquireRepoLock, + releaseRepoLock, + }); + /** * Maximum time the hold-queue will wait for an active analysis job to complete. * Must stay in sync with the frontend's `fetchRepoInfo({ awaitAnalysis: true })` timeout. @@ -994,6 +1010,14 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => } } + // 2b. Delete the uploaded repo dir if entry.path lives under + // UPLOAD_ROOT. Drive this off entry.path (not a name-rederived dir) so + // a same-named clone is never affected. + const resolvedEntry = path.resolve(entry.path); + if (resolvedEntry === UPLOAD_ROOT || resolvedEntry.startsWith(UPLOAD_ROOT + path.sep)) { + await fs.rm(resolvedEntry, { recursive: true, force: true }).catch(() => {}); + } + // 3. Unregister from the global registry const { unregisterRepo } = await import('../storage/repo-manager.js'); await unregisterRepo(entry.path); @@ -1428,229 +1452,115 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // ── Analyze API ────────────────────────────────────────────────────── // POST /api/analyze — start a new analysis job - app.post('/api/analyze', createRouteLimiter({ limit: 10 }), async (req, res) => { - try { - const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings } = req.body; + app.post( + '/api/analyze', + createRouteLimiter({ limit: 10 }), + requireLocalhostOrigin, + async (req, res) => { + try { + const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings } = req.body; - // Input type validation - if (repoUrl !== undefined && typeof repoUrl !== 'string') { - res.status(400).json({ error: '"url" must be a string' }); - return; - } - if (repoLocalPath !== undefined && typeof repoLocalPath !== 'string') { - res.status(400).json({ error: '"path" must be a string' }); - return; - } + // Input type validation + if (repoUrl !== undefined && typeof repoUrl !== 'string') { + res.status(400).json({ error: '"url" must be a string' }); + return; + } + if (repoLocalPath !== undefined && typeof repoLocalPath !== 'string') { + res.status(400).json({ error: '"path" must be a string' }); + return; + } - if (!repoUrl && !repoLocalPath) { - res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' }); - return; - } + if (!repoUrl && !repoLocalPath) { + res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' }); + return; + } - // Path validation: require absolute path, reject traversal (e.g. /tmp/../etc/passwd) - if (repoLocalPath) { - if (!path.isAbsolute(repoLocalPath)) { + // Path validation. The previous `normalize !== resolve` guard was inert + // (both collapse `..` identically) and only false-rejected trailing + // slashes, so it is dropped. Analyzing a local path the operator names + // is the tool's intended capability (same as the CLI); the dangerous + // part was cross-origin reach, which is closed by requireLocalhostOrigin + // on this route. We only require an absolute path here and let the + // analyze worker surface a clear error if it does not exist. (We do NOT + // realpath/stat the path in-route: that would be a user-controlled + // filesystem read — CodeQL js/path-injection — for no security gain.) + if (repoLocalPath && !path.isAbsolute(repoLocalPath)) { res.status(400).json({ error: '"path" must be an absolute path' }); return; } - if (path.normalize(repoLocalPath) !== path.resolve(repoLocalPath)) { - res.status(400).json({ error: '"path" must not contain traversal sequences' }); + + const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath }); + + // If job was already running (dedup), just return its id + if (job.status !== 'queued') { + res.status(202).json({ jobId: job.id, status: job.status }); return; } - } - const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath }); + // Mark as active synchronously to prevent race with concurrent requests + jobManager.updateJob(job.id, { status: 'cloning' }); - // If job was already running (dedup), just return its id - if (job.status !== 'queued') { - res.status(202).json({ jobId: job.id, status: job.status }); - return; - } + // Start async work — don't await + (async () => { + let targetPath = repoLocalPath; + try { + // Clone if URL provided + if (repoUrl && !repoLocalPath) { + const repoName = extractRepoName(repoUrl); + targetPath = getCloneDir(repoName); - // Mark as active synchronously to prevent race with concurrent requests - jobManager.updateJob(job.id, { status: 'cloning' }); + jobManager.updateJob(job.id, { + status: 'cloning', + repoName, + progress: { phase: 'cloning', percent: 0, message: `Cloning ${repoUrl}...` }, + }); - // Start async work — don't await - (async () => { - let targetPath = repoLocalPath; - try { - // Clone if URL provided - if (repoUrl && !repoLocalPath) { - const repoName = extractRepoName(repoUrl); - targetPath = getCloneDir(repoName); + await cloneOrPull(repoUrl, targetPath, (progress) => { + jobManager.updateJob(job.id, { + progress: { phase: progress.phase, percent: 5, message: progress.message }, + }); + }); + } + if (!targetPath) { + throw new Error('No target path resolved'); + } + + launchAnalysisWorker(job, targetPath, { force, embeddings, dropEmbeddings }); + } catch (err: any) { + if (targetPath) releaseRepoLock(getStoragePath(targetPath)); jobManager.updateJob(job.id, { - status: 'cloning', - repoName, - progress: { phase: 'cloning', percent: 0, message: `Cloning ${repoUrl}...` }, - }); - - await cloneOrPull(repoUrl, targetPath, (progress) => { - jobManager.updateJob(job.id, { - progress: { phase: progress.phase, percent: 5, message: progress.message }, - }); + status: 'failed', + error: err.message || 'Analysis failed', }); } + })(); - if (!targetPath) { - throw new Error('No target path resolved'); - } - - // Acquire shared repo lock (keyed on storagePath to match embed handler) - const analyzeLockKey = getStoragePath(targetPath); - const lockErr = acquireRepoLock(analyzeLockKey); - if (lockErr) { - jobManager.updateJob(job.id, { status: 'failed', error: lockErr }); - return; - } - - jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' }); - - // ── Worker fork with auto-retry ────────────────────────────── - // - // Forks a child process with 8GB heap. If the worker crashes - // (OOM, native addon segfault, etc.), it retries up to - // MAX_WORKER_RETRIES times with exponential backoff before - // marking the job as permanently failed. - // - // In dev mode (tsx), registers the tsx ESM hook via a file:// - // URL so the child can compile TypeScript on-the-fly. - - const MAX_WORKER_RETRIES = 2; - const callerPath = fileURLToPath(import.meta.url); - const isDev = callerPath.endsWith('.ts'); - const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js'; - const workerPath = path.join(path.dirname(callerPath), workerFile); - const tsxHookArgs: string[] = isDev - ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] - : []; - - const forkWorker = () => { - const currentJob = jobManager.getJob(job.id); - if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed') - return; - - const child = fork(workerPath, [], { - execArgv: [...tsxHookArgs, '--max-old-space-size=8192'], - stdio: ['ignore', 'pipe', 'pipe', 'ipc'], - }); - - // Capture stderr for crash diagnostics - let stderrChunks = ''; - child.stderr?.on('data', (chunk: Buffer) => { - stderrChunks += chunk.toString(); - if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096); - }); - - child.on('message', (msg: any) => { - if (msg.type === 'progress') { - jobManager.updateJob(job.id, { - status: 'analyzing', - progress: { phase: msg.phase, percent: msg.percent, message: msg.message }, - }); - } else if (msg.type === 'complete') { - releaseRepoLock(analyzeLockKey); - // Reinitialize backend BEFORE marking complete — ensures the new - // repo is queryable when the client receives the SSE complete event. - backend - .init() - .then(() => { - jobManager.updateJob(job.id, { - status: 'complete', - repoName: msg.result.repoName, - }); - }) - .catch((err) => { - logger.error({ err }, 'backend.init() failed after analyze:'); - jobManager.updateJob(job.id, { - status: 'failed', - error: 'Server failed to reload after analysis. Try again.', - }); - }); - } else if (msg.type === 'error') { - releaseRepoLock(analyzeLockKey); - jobManager.updateJob(job.id, { - status: 'failed', - error: msg.message, - }); - } - }); - - child.on('error', (err) => { - releaseRepoLock(analyzeLockKey); - jobManager.updateJob(job.id, { - status: 'failed', - error: `Worker process error: ${err.message}`, - }); - }); - - child.on('exit', (code) => { - const j = jobManager.getJob(job.id); - if (!j || j.status === 'complete' || j.status === 'failed') return; - - // Worker crashed — attempt retry if under the limit - if (j.retryCount < MAX_WORKER_RETRIES) { - j.retryCount++; - const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s - const lastErr = stderrChunks.trim().split('\n').pop() || ''; - logger.warn( - `Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` + - (lastErr ? `: ${lastErr}` : ''), - ); - jobManager.updateJob(job.id, { - status: 'analyzing', - progress: { - phase: 'retrying', - percent: j.progress.percent, - message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`, - }, - }); - stderrChunks = ''; - setTimeout(forkWorker, delay); - } else { - // Exhausted retries — permanent failure - releaseRepoLock(analyzeLockKey); - jobManager.updateJob(job.id, { - status: 'failed', - error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`, - }); - } - }); - - // Register child for cancellation + timeout tracking - jobManager.registerChild(job.id, child); - - // Send start command to child - child.send({ - type: 'start', - repoPath: targetPath, - options: { - force: !!force, - embeddings: !!embeddings, - dropEmbeddings: !!dropEmbeddings, - }, - }); - }; - - forkWorker(); - } catch (err: any) { - if (targetPath) releaseRepoLock(getStoragePath(targetPath)); - jobManager.updateJob(job.id, { - status: 'failed', - error: err.message || 'Analysis failed', - }); + res.status(202).json({ jobId: job.id, status: job.status }); + } catch (err: any) { + if (err.message?.includes('already in progress')) { + res.status(409).json({ error: err.message }); + } else { + res.status(500).json({ error: err.message || 'Failed to start analysis' }); } - })(); - - res.status(202).json({ jobId: job.id, status: job.status }); - } catch (err: any) { - if (err.message?.includes('already in progress')) { - res.status(409).json({ error: err.message }); - } else { - res.status(500).json({ error: err.message || 'Failed to start analysis' }); } - } - }); + }, + ); + + // POST /api/analyze/upload — analyze a browser folder upload. + // Securely ingests the multipart upload into a sandbox, promotes it to a + // persistent dir, and analyzes it via the shared job/worker machinery. + // localhost-only (no cross-origin write reach) + conservative rate limit. + app.post( + '/api/analyze/upload', + createRouteLimiter({ limit: 5 }), + requireLocalhostOrigin, + createAnalyzeUploadHandler({ + createJob: (params) => jobManager.createJob(params), + launch: (job, targetPath, opts) => launchAnalysisWorker(job, targetPath, opts), + failJob: (jobId, error) => jobManager.updateJob(jobId, { status: 'failed', error }), + }), + ); // GET /api/analyze/:jobId — poll job status app.get('/api/analyze/:jobId', (req, res) => { diff --git a/gitnexus/src/server/git-clone.ts b/gitnexus/src/server/git-clone.ts index d92e9c28f..eae13b94e 100644 --- a/gitnexus/src/server/git-clone.ts +++ b/gitnexus/src/server/git-clone.ts @@ -20,7 +20,7 @@ const CLONE_ROOT = path.resolve(path.join(os.homedir(), '.gitnexus', 'repos')); // Rejecting anything else (including `..`, `/`, `\`, shell metacharacters) // guarantees getCloneDir(repoName) cannot escape CLONE_ROOT regardless of // how the caller derived repoName. -const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; +export const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; /** * Extract the repository name from a git URL (HTTPS or SSH). diff --git a/gitnexus/src/server/middleware.ts b/gitnexus/src/server/middleware.ts new file mode 100644 index 000000000..73d3edab7 --- /dev/null +++ b/gitnexus/src/server/middleware.ts @@ -0,0 +1,29 @@ +/** + * Shared Express route guards (alongside createRouteLimiter in validation.ts). + */ + +import type { Request, Response } from 'express'; + +/** + * Restrict a route to localhost browser origins. Non-browser requests (no + * Origin header, e.g. curl / the CLI) pass through. This closes cross-origin + * reach (the allow-listed public deploy + Private Network Access) to write + * routes without affecting read routes. + */ +export function requireLocalhostOrigin(req: Request, res: Response, next: () => void): void { + const origin = req.headers.origin; + if (origin === undefined) { + next(); + return; + } + try { + const hostname = new URL(origin).hostname; + if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') { + next(); + return; + } + } catch { + /* malformed origin → reject */ + } + res.status(403).json({ error: 'This endpoint is restricted to localhost origins' }); +} diff --git a/gitnexus/src/server/upload-ingest.ts b/gitnexus/src/server/upload-ingest.ts new file mode 100644 index 000000000..2dd52b0e6 --- /dev/null +++ b/gitnexus/src/server/upload-ingest.ts @@ -0,0 +1,320 @@ +/** + * Secure ingestion of a browser folder upload (multipart/form-data). + * + * Replaces the path-injection-prone GET /api/fs/list directory listing. The + * client streams the selected files plus a JSON `manifest` of their + * webkitRelativePaths; we write each into an mkdtemp staging dir under + * UPLOAD_ROOT with PROVABLE containment (resolve-then-contain), hard resource + * caps, and guaranteed cleanup on every failure/abort path. No client value + * ever reaches a filesystem READ — the server only writes into a sandbox it + * created, then hands that sandbox to the analysis pipeline. + * + * Security references: CodeQL js/path-injection (resolve + startsWith(root+sep)), + * OWASP File Upload / Path Traversal. + */ + +import path from 'path'; +import fs from 'fs'; +import fsp from 'fs/promises'; +import type { IncomingMessage } from 'http'; +import busboy from 'busboy'; +import { UPLOAD_ROOT, STAGING_PREFIX } from './upload-paths.js'; +import { BadRequestError } from './validation.js'; + +export interface IngestLimits { + /** Aggregate bytes across all files (busboy has no aggregate limit). */ + maxTotalBytes: number; + /** Per-file byte cap. */ + maxFileBytes: number; + /** Maximum number of files. */ + maxFiles: number; + /** Maximum multipart parts (files + fields). */ + maxParts: number; + /** Maximum directories created (inode-exhaustion guard). */ + maxDirs: number; + /** Maximum size of the manifest field. */ + maxFieldBytes: number; +} + +export const DEFAULT_INGEST_LIMITS: IngestLimits = { + maxTotalBytes: 250 * 1024 * 1024, + maxFileBytes: 25 * 1024 * 1024, + maxFiles: 20000, + maxParts: 20100, + maxDirs: 50000, + maxFieldBytes: 2 * 1024 * 1024, +}; + +const MAX_PATH_DEPTH = 64; +const MAX_PATH_LENGTH = 4096; + +export interface IngestResult { + /** Absolute path to the populated staging directory (realpath-canonical). */ + stageRoot: string; + fileCount: number; + totalBytes: number; + /** First path segment shared by the uploaded tree (the picked folder). */ + topLevelName: string; +} + +/** + * Resolve a client-provided relative path to an absolute destination PROVABLY + * contained within `stageRoot`. Throws BadRequestError on any unsafe input. + * This is the load-bearing path-traversal-on-write control; keep it pure and + * unit-tested. + */ +export function resolveContainedDest(stageRoot: string, rel: unknown): string { + if (typeof rel !== 'string' || rel.length === 0) { + throw new BadRequestError('Invalid upload path'); + } + if (rel.length > MAX_PATH_LENGTH) { + throw new BadRequestError('Upload path too long'); + } + // webkitRelativePath is always relative; a leading slash is absolute/hostile. + if (rel.startsWith('/')) { + throw new BadRequestError('Invalid upload path'); + } + // Browsers emit forward slashes only; a NUL byte or backslash is hostile. + if (rel.includes('\u0000') || rel.includes('\\')) { + throw new BadRequestError('Invalid upload path'); + } + const rawSegments = rel.split('/').filter((s) => s.length > 0); + if (rawSegments.length === 0 || rawSegments.length > MAX_PATH_DEPTH) { + throw new BadRequestError('Invalid upload path'); + } + const segments: string[] = []; + for (const seg of rawSegments) { + // Normalize so NFC/NFD variants don't collide silently on case/unicode + // -folding filesystems (macOS/Windows). + const s = seg.normalize('NFC'); + if (s === '.' || s === '..') { + throw new BadRequestError('Upload path must not contain traversal segments'); + } + segments.push(s); + } + const dest = path.resolve(stageRoot, segments.join(path.sep)); + // Suffix path.sep so a sibling prefix (/sandbox-evil vs /sandbox) can't pass. + if (dest !== stageRoot && !dest.startsWith(stageRoot + path.sep)) { + throw new BadRequestError('Upload path escapes the sandbox'); + } + return dest; +} + +interface DirState { + dirCount: number; + limits: IngestLimits; +} + +/** + * Create the parent directories of `destFile` one segment at a time, asserting + * after each `mkdir` that the segment is a real directory (not a symlink + * swapped in mid-stream) still inside `stageRoot`. Counts created dirs against + * the inode-exhaustion cap. + */ +function mkdirContained(stageRoot: string, destFile: string, state: DirState): void { + const parent = path.dirname(destFile); + const relParent = path.relative(stageRoot, parent); + if (relParent === '' || relParent === '.') return; + const segs = relParent.split(path.sep).filter(Boolean); + let cur = stageRoot; + for (const seg of segs) { + cur = path.join(cur, seg); + let made = false; + try { + fs.mkdirSync(cur); + made = true; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + } + const st = fs.lstatSync(cur); + if (st.isSymbolicLink() || !st.isDirectory()) { + throw new BadRequestError('Upload path escapes the sandbox'); + } + if (made) { + state.dirCount++; + if (state.dirCount > state.limits.maxDirs) { + throw new BadRequestError('Too many directories in upload', 413); + } + } + } +} + +export interface IngestOptions { + /** Override the staging parent dir (defaults to UPLOAD_ROOT; for tests). */ + root?: string; +} + +/** + * Parse and securely write a multipart folder upload into a fresh staging + * directory under UPLOAD_ROOT. Resolves with the populated staging dir, or + * rejects with a BadRequestError (status 400/413) after removing the staging + * dir. The caller owns promotion + cleanup of the returned `stageRoot`. + */ +export async function ingestUpload( + req: IncomingMessage, + limitsOverride?: Partial, + opts: IngestOptions = {}, +): Promise { + const limits = { ...DEFAULT_INGEST_LIMITS, ...limitsOverride }; + const uploadRoot = opts.root ?? UPLOAD_ROOT; + await fsp.mkdir(uploadRoot, { recursive: true }); + // mkdtemp creates the dir mode 0o700 (owner-only); realpath canonicalizes + // the root so the containment prefix check is exact. + const stageRoot = await fsp.realpath(await fsp.mkdtemp(path.join(uploadRoot, STAGING_PREFIX))); + + let cleaned = false; + const cleanup = async (): Promise => { + if (cleaned) return; + cleaned = true; + await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => {}); + }; + + return new Promise((resolve, reject) => { + let settled = false; + let manifest: string[] | null = null; + let fileIndex = 0; + let fileCount = 0; + let totalBytes = 0; + let topLevelName = ''; + const dirState: DirState = { dirCount: 0, limits }; + const writePromises: Promise[] = []; + + const bb = busboy({ + headers: req.headers, + limits: { + fileSize: limits.maxFileBytes, + files: limits.maxFiles, + parts: limits.maxParts, + fields: 10, + fieldNameSize: 200, + fieldSize: limits.maxFieldBytes, + headerPairs: 2000, + }, + }); + + const fail = (err: Error): void => { + if (settled) return; + settled = true; + try { + req.unpipe(bb); + } catch { + /* ignore */ + } + try { + req.resume(); // drain remaining body so the socket isn't left hanging + } catch { + /* ignore */ + } + void cleanup().finally(() => reject(err)); + }; + + bb.on('field', (name: string, val: string) => { + if (name !== 'manifest') return; + try { + const parsed = JSON.parse(val); + if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === 'string')) { + return fail(new BadRequestError('Invalid manifest')); + } + manifest = parsed as string[]; + } catch { + fail(new BadRequestError('Invalid manifest')); + } + }); + + bb.on('file', (_name: string, stream: NodeJS.ReadableStream, _info: unknown) => { + if (settled) { + stream.resume(); + return; + } + if (manifest === null) { + // The manifest field MUST arrive before any file part. + stream.resume(); + return fail(new BadRequestError('Manifest must precede file parts')); + } + const idx = fileIndex++; + const rel = manifest[idx]; + let dest: string; + try { + dest = resolveContainedDest(stageRoot, rel); + // A folder upload is exactly one top-level directory: every entry must + // have ≥2 segments and share the same first segment. This rejects a + // bare file at the root (which would make the promote target a file) + // and a multi-top manifest (which would silently drop all but the + // first folder). Validated here, before any job is created. + const segs = String(rel) + .split('/') + .filter((s) => s.length > 0); + const firstSeg = (segs[0] ?? '').normalize('NFC'); + if (!topLevelName) { + topLevelName = firstSeg; + } + if (segs.length < 2 || firstSeg !== topLevelName) { + throw new BadRequestError('Upload must be a single folder of files'); + } + mkdirContained(stageRoot, dest, dirState); + } catch (err) { + stream.resume(); + return fail(err as Error); + } + fileCount++; + const ws = fs.createWriteStream(dest, { flags: 'wx' }); + const p = new Promise((res, rej) => { + stream.on('data', (chunk: Buffer) => { + totalBytes += chunk.length; + if (totalBytes > limits.maxTotalBytes) { + stream.unpipe(ws); + ws.destroy(); + rej(new BadRequestError('Upload exceeds total size limit', 413)); + } + }); + stream.on('limit', () => { + ws.destroy(); + rej(new BadRequestError('File exceeds size limit', 413)); + }); + stream.on('error', rej); + ws.on('error', rej); + ws.on('finish', () => res()); + stream.pipe(ws); + }); + writePromises.push(p); + p.catch(fail); + }); + + bb.on('filesLimit', () => fail(new BadRequestError('Too many files in upload', 413))); + bb.on('partsLimit', () => fail(new BadRequestError('Too many parts in upload', 413))); + bb.on('fieldsLimit', () => fail(new BadRequestError('Too many fields in upload'))); + bb.on('error', (err: unknown) => + fail(err instanceof Error ? err : new BadRequestError('Upload parse error')), + ); + + bb.on('close', () => { + if (settled) return; + Promise.all(writePromises) + .then(() => { + if (settled) return; + if (manifest === null) return fail(new BadRequestError('Missing manifest')); + if (fileCount === 0) return fail(new BadRequestError('Empty upload')); + if (fileCount !== manifest.length) { + return fail(new BadRequestError('Manifest/file count mismatch')); + } + if (!topLevelName) { + return fail(new BadRequestError('Could not determine upload folder name')); + } + settled = true; + resolve({ stageRoot, fileCount, totalBytes, topLevelName }); + }) + .catch(() => { + /* a write rejected → fail() already invoked via p.catch */ + }); + }); + + req.on('aborted', () => { + if (!settled) fail(new BadRequestError('Upload aborted')); + }); + req.on('error', (err: unknown) => + fail(err instanceof Error ? err : new BadRequestError('Request error')), + ); + + req.pipe(bb); + }); +} diff --git a/gitnexus/src/server/upload-paths.ts b/gitnexus/src/server/upload-paths.ts new file mode 100644 index 000000000..ef957c85b --- /dev/null +++ b/gitnexus/src/server/upload-paths.ts @@ -0,0 +1,58 @@ +/** + * Upload working-directory paths. + * + * Browser folder uploads are written into ~/.gitnexus/uploads/{name}/ — a + * sibling of the clone root (git-clone.ts CLONE_ROOT) — so an uploaded repo + * persists and behaves like a cloned one (the graph UI's /api/file reads its + * files after analysis, and DELETE /api/repo removes it). Staging happens in + * an mkdtemp dir *under* UPLOAD_ROOT so the promote rename stays on one + * filesystem and remains atomic (a rename from os.tmpdir() could trip EXDEV — + * the exact Docker case this feature targets; see bridge-db.ts for the same + * anchored-staging pattern). + */ + +import path from 'path'; +import os from 'os'; +import { sanitizeRepoName } from '../storage/git.js'; +import { REPO_NAME_PATTERN } from './git-clone.js'; + +/** Root directory for all uploaded repositories. Targets must resolve inside this. */ +export const UPLOAD_ROOT = path.resolve(path.join(os.homedir(), '.gitnexus', 'uploads')); + +/** Prefix for per-upload staging directories created under UPLOAD_ROOT. */ +export const STAGING_PREFIX = '.staging-'; + +/** + * Get the upload target directory for a repo name. + * + * Re-validates at the boundary (callers may derive the name from an untrusted + * manifest). Rejects `.`, `..`, the `'unknown'` sentinel that sanitizeRepoName + * emits for un-nameable inputs, names beginning with `.` (which would collide + * with the `.staging-` prefix), and anything outside the safe charset. + */ +export function getUploadDir(repoName: string): string { + if ( + !repoName || + repoName === '.' || + repoName === '..' || + repoName === 'unknown' || + repoName.startsWith('.') || + !REPO_NAME_PATTERN.test(repoName) + ) { + throw new Error('Invalid repository name'); + } + return path.join(UPLOAD_ROOT, repoName); +} + +/** + * Derive a filesystem-safe upload directory name from the manifest's + * top-level folder. Returns null when the name is un-nameable (so the caller + * rejects with 400 rather than colliding everyone on `UPLOAD_ROOT/unknown`). + */ +export function deriveUploadName(topLevelName: string): string | null { + const safe = sanitizeRepoName(topLevelName); + if (safe === 'unknown' || safe === '.' || safe === '..' || safe.startsWith('.')) { + return null; + } + return safe; +} diff --git a/gitnexus/src/server/upload-sweep.ts b/gitnexus/src/server/upload-sweep.ts new file mode 100644 index 000000000..46dcd4aed --- /dev/null +++ b/gitnexus/src/server/upload-sweep.ts @@ -0,0 +1,67 @@ +/** + * Backstop cleanup for abandoned upload staging directories. + * + * A crashed/killed process can leave a `.staging-*` directory under + * UPLOAD_ROOT (the normal path removes it on success/failure/abort). This + * sweep, run once at server startup, removes staging dirs older than a + * threshold. Promoted upload dirs are persistent registered repos (like + * clones) and are NOT touched here — they are removed via DELETE /api/repo. + */ + +import path from 'path'; +import fsp from 'fs/promises'; +import { UPLOAD_ROOT, STAGING_PREFIX } from './upload-paths.js'; + +export interface SweepOptions { + /** Remove staging dirs older than this (default 6h). */ + maxAgeMs?: number; + /** Override the root to sweep (defaults to UPLOAD_ROOT; for tests). */ + root?: string; + /** Clock injection for tests. */ + now?: number; +} + +export async function sweepStaleUploads(opts: SweepOptions = {}): Promise<{ removed: string[] }> { + const maxAgeMs = opts.maxAgeMs ?? 6 * 60 * 60 * 1000; + const root = opts.root ?? UPLOAD_ROOT; + const now = opts.now ?? Date.now(); + const removed: string[] = []; + + let entries; + try { + entries = await fsp.readdir(root, { withFileTypes: true }); + } catch { + return { removed }; // root does not exist yet — nothing to sweep + } + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const full = path.join(root, entry.name); + try { + const st = await fsp.stat(full); + if (now - st.mtimeMs <= maxAgeMs) continue; // recent — keep + + if (entry.name.startsWith(STAGING_PREFIX)) { + // Transient staging dir orphaned by a crash — always removable. + await fsp.rm(full, { recursive: true, force: true }).catch(() => {}); + removed.push(full); + } else { + // Promoted upload dir. A successfully-analyzed (registered) repo always + // has a `.gitnexus` index inside it; a stale promoted dir WITHOUT one is + // an orphan from an analysis that failed before registering — remove it. + const hasIndex = await fsp + .access(path.join(full, '.gitnexus')) + .then(() => true) + .catch(() => false); + if (!hasIndex) { + await fsp.rm(full, { recursive: true, force: true }).catch(() => {}); + removed.push(full); + } + } + } catch { + /* stat race — skip */ + } + } + + return { removed }; +} diff --git a/gitnexus/test/unit/api-analyze-upload.test.ts b/gitnexus/test/unit/api-analyze-upload.test.ts new file mode 100644 index 000000000..af3bbfd23 --- /dev/null +++ b/gitnexus/test/unit/api-analyze-upload.test.ts @@ -0,0 +1,289 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { Readable } from 'node:stream'; +import type { IncomingMessage } from 'node:http'; +import { createAnalyzeUploadHandler } from '../../src/server/analyze-upload.js'; +import { requireLocalhostOrigin } from '../../src/server/middleware.js'; + +const BOUNDARY = '----gitnexusuploadtest'; + +function multipart( + parts: Array<{ name: string; value?: string; filename?: string; data?: Buffer }>, +): { body: Buffer; headers: Record } { + const chunks: Buffer[] = []; + for (const p of parts) { + chunks.push(Buffer.from(`--${BOUNDARY}\r\n`)); + if (p.filename !== undefined) { + chunks.push( + Buffer.from( + `Content-Disposition: form-data; name="${p.name}"; filename="${p.filename}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n`, + ), + ); + chunks.push(p.data ?? Buffer.alloc(0)); + chunks.push(Buffer.from('\r\n')); + } else { + chunks.push(Buffer.from(`Content-Disposition: form-data; name="${p.name}"\r\n\r\n`)); + chunks.push(Buffer.from(p.value ?? '')); + chunks.push(Buffer.from('\r\n')); + } + } + chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`)); + return { + body: Buffer.concat(chunks), + headers: { 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }, + }; +} + +function mockReq(parts: Parameters[0]): IncomingMessage { + const { body, headers } = multipart(parts); + const r = Readable.from([body]) as unknown as IncomingMessage & { headers: typeof headers }; + r.headers = headers; + return r; +} + +interface MockRes { + statusCode: number; + body: unknown; + status: (c: number) => MockRes; + json: (b: unknown) => MockRes; +} +function mockRes(): MockRes { + const res = { statusCode: 0, body: undefined as unknown } as MockRes; + res.status = (c: number) => { + res.statusCode = c; + return res; + }; + res.json = (b: unknown) => { + res.body = b; + return res; + }; + return res; +} + +// Track promoted upload dirs created by the real ingest+promote so we clean up. +const promoted: string[] = []; +afterEach(async () => { + while (promoted.length) { + await fs.rm(promoted.pop()!, { recursive: true, force: true }).catch(() => {}); + } +}); + +function uniqueTop(): string { + return `uptest_${Math.floor(Math.random() * 1e9).toString(36)}`; +} + +describe('createAnalyzeUploadHandler', () => { + it('ingests, promotes the inner folder, and launches analysis (202)', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => ({ id: 'job-1', status: 'queued' })); + const launch = vi.fn((_j, dir: string) => promoted.push(dir)); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/a.js`, `${top}/sub/b.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('alpha') }, + { name: 'files', filename: 'blob', data: Buffer.from('beta') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(202); + expect((res.body as { jobId: string }).jobId).toBe('job-1'); + expect(createJob).toHaveBeenCalledOnce(); + expect(launch).toHaveBeenCalledOnce(); + const dir = launch.mock.calls[0][1] as string; + const opts = launch.mock.calls[0][2] as { registryName: string }; + // Inner folder promoted: contents live directly under the upload dir. + expect(await fs.readFile(path.join(dir, 'a.js'), 'utf8')).toBe('alpha'); + expect(await fs.readFile(path.join(dir, 'sub', 'b.js'), 'utf8')).toBe('beta'); + expect(opts.registryName).toBe(top); + expect(createJob.mock.calls[0][0].repoPath).toBe(dir); + }); + + it('maps a busy job (createJob throws "already in progress") to 409 and promotes nothing', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => { + throw new Error('Analysis already in progress for another repository'); + }); + const launch = vi.fn((_j, dir: string) => promoted.push(dir)); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/a.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(409); + expect(launch).not.toHaveBeenCalled(); + // Nothing promoted onto disk. + const { UPLOAD_ROOT } = await import('../../src/server/upload-paths.js'); + await expect(fs.access(path.join(UPLOAD_ROOT, top))).rejects.toBeTruthy(); + }); + + it('rejects a traversal path in the manifest (400) without launching', async () => { + const createJob = vi.fn(() => ({ id: 'j', status: 'queued' })); + const launch = vi.fn(); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify(['../escape.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(400); + expect(createJob).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + }); + + it('rejects an un-nameable top folder (Windows-reserved → 400)', async () => { + const createJob = vi.fn(() => ({ id: 'j', status: 'queued' })); + const launch = vi.fn(); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify(['CON/a.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(400); + expect(launch).not.toHaveBeenCalled(); + }); + + it('strips a crafted .gitnexus index from the promoted upload', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => ({ id: 'job-x', status: 'queued' })); + const launch = vi.fn((_j, dir: string) => promoted.push(dir)); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/.gitnexus/meta.json`, `${top}/a.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('{"evil":true}') }, + { name: 'files', filename: 'blob', data: Buffer.from('real') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(202); + const dir = launch.mock.calls[0][1] as string; + await expect(fs.access(path.join(dir, '.gitnexus'))).rejects.toBeTruthy(); + expect(await fs.readFile(path.join(dir, 'a.js'), 'utf8')).toBe('real'); + }); + + it('rejects a single-segment manifest before creating a job (no slot taken)', async () => { + const createJob = vi.fn(() => ({ id: 'j', status: 'queued' })); + const launch = vi.fn(); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify(['loosefile.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(400); + expect(createJob).not.toHaveBeenCalled(); // slot never taken → no wedge + expect(launch).not.toHaveBeenCalled(); + }); + + it('rejects a multi-top-folder manifest (would silently drop folders)', async () => { + const createJob = vi.fn(() => ({ id: 'j', status: 'queued' })); + const launch = vi.fn(); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify(['aaa/x.js', 'bbb/y.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('1') }, + { name: 'files', filename: 'blob', data: Buffer.from('2') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(400); + expect(createJob).not.toHaveBeenCalled(); + }); + + it('releases the single slot (failJob) when a step fails after createJob', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => ({ id: 'job-fail', status: 'queued' })); + // launch throws AFTER createJob + promote — the slot must be released. + const launch = vi.fn((_j, dir: string) => { + promoted.push(dir); + throw new Error('worker fork blew up'); + }); + const failJob = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch, failJob }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/a.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(createJob).toHaveBeenCalledOnce(); + expect(failJob).toHaveBeenCalledWith('job-fail', expect.any(String)); + expect(res.statusCode).toBe(500); + }); +}); + +describe('requireLocalhostOrigin', () => { + function call(origin: string | undefined): { passed: boolean; status: number } { + let passed = false; + let status = 0; + const req = { headers: origin === undefined ? {} : { origin } } as never; + const res = { + status: (c: number) => { + status = c; + return { json: () => {} }; + }, + } as never; + requireLocalhostOrigin(req, res, () => { + passed = true; + }); + return { passed, status }; + } + + it('passes localhost / 127.0.0.1 / no-origin', () => { + expect(call('http://localhost:5173').passed).toBe(true); + expect(call('http://127.0.0.1:4747').passed).toBe(true); + expect(call(undefined).passed).toBe(true); + }); + + it('rejects a public/cross origin with 403', () => { + const r = call('https://gitnexus.vercel.app'); + expect(r.passed).toBe(false); + expect(r.status).toBe(403); + }); +}); diff --git a/gitnexus/test/unit/rate-limit.test.ts b/gitnexus/test/unit/rate-limit.test.ts index 6a6d83a25..285a8f8c3 100644 --- a/gitnexus/test/unit/rate-limit.test.ts +++ b/gitnexus/test/unit/rate-limit.test.ts @@ -241,7 +241,9 @@ describe('production routes — rate-limit middleware wiring', () => { }); it('POST /api/analyze is wired with createRouteLimiter', () => { - expect(apiSource).toMatch(/app\.post\('\/api\/analyze',\s*createRouteLimiter\(/); + // Tolerate Prettier wrapping the registration across lines (it does once + // the route carries extra middleware like requireLocalhostOrigin). + expect(apiSource).toMatch(/app\.post\(\s*'\/api\/analyze',\s*createRouteLimiter\(/); }); it('POST /api/embed is wired with createRouteLimiter', () => { diff --git a/gitnexus/test/unit/upload-ingest.test.ts b/gitnexus/test/unit/upload-ingest.test.ts new file mode 100644 index 000000000..353d26862 --- /dev/null +++ b/gitnexus/test/unit/upload-ingest.test.ts @@ -0,0 +1,178 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import { Readable } from 'node:stream'; +import type { IncomingMessage } from 'node:http'; +import { + resolveContainedDest, + ingestUpload, + DEFAULT_INGEST_LIMITS, +} from '../../src/server/upload-ingest.js'; +import { STAGING_PREFIX } from '../../src/server/upload-paths.js'; + +// ── resolveContainedDest (pure sanitizer — the load-bearing control) ────────── + +describe('resolveContainedDest', () => { + const ROOT = path.resolve('/tmp/gitnexus-sandbox'); + + it('contains a legitimate nested path under the root', () => { + expect(resolveContainedDest(ROOT, 'myrepo/src/index.js')).toBe( + path.join(ROOT, 'myrepo', 'src', 'index.js'), + ); + }); + + it('allows spaces in names (regression: NUL check must not reject spaces)', () => { + expect(resolveContainedDest(ROOT, 'my repo/a b.js')).toBe(path.join(ROOT, 'my repo', 'a b.js')); + }); + + it.each([ + ['parent traversal', '../../etc/passwd'], + ['mid traversal', 'a/../../b'], + ['dot-dot segment', 'a/../b'], + ['dot segment', 'a/./b'], + ['absolute path', '/etc/shadow'], + ['backslash', 'a\\b'], + ['empty', ''], + ])('rejects %s', (_label, rel) => { + expect(() => resolveContainedDest(ROOT, rel)).toThrow(); + }); + + it('rejects a NUL byte', () => { + expect(() => resolveContainedDest(ROOT, `a${String.fromCharCode(0)}b/x.js`)).toThrow(); + }); + + it('rejects non-string input', () => { + // @ts-expect-error testing runtime guard + expect(() => resolveContainedDest(ROOT, ['a', 'b'])).toThrow(); + }); + + it('rejects an over-deep path (>64 segments)', () => { + const deep = Array.from({ length: 70 }, (_, i) => `d${i}`).join('/') + '/f.js'; + expect(() => resolveContainedDest(ROOT, deep)).toThrow(); + }); + + it('rejects an over-long path (>4096 chars)', () => { + const long = 'a/'.repeat(2100) + 'f.js'; + expect(() => resolveContainedDest(ROOT, long)).toThrow(); + }); + + it('rejects a sibling-prefix escape (root + sep, not bare startsWith)', () => { + // A rel that would resolve to a sibling dir sharing the root's string prefix. + expect(() => resolveContainedDest(ROOT, '../gitnexus-sandbox-evil/x.js')).toThrow(); + }); +}); + +// ── ingestUpload (multipart streaming + containment + caps + cleanup) ────────── + +const BOUNDARY = '----gitnexustestboundary'; + +function multipart( + parts: Array<{ name: string; value?: string; filename?: string; data?: Buffer }>, +): { body: Buffer; headers: Record } { + const chunks: Buffer[] = []; + for (const p of parts) { + chunks.push(Buffer.from(`--${BOUNDARY}\r\n`)); + if (p.filename !== undefined) { + chunks.push( + Buffer.from( + `Content-Disposition: form-data; name="${p.name}"; filename="${p.filename}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n`, + ), + ); + chunks.push(p.data ?? Buffer.alloc(0)); + chunks.push(Buffer.from('\r\n')); + } else { + chunks.push(Buffer.from(`Content-Disposition: form-data; name="${p.name}"\r\n\r\n`)); + chunks.push(Buffer.from(p.value ?? '')); + chunks.push(Buffer.from('\r\n')); + } + } + chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`)); + return { + body: Buffer.concat(chunks), + headers: { 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }, + }; +} + +function mockReq(body: Buffer, headers: Record): IncomingMessage { + const r = Readable.from([body]) as unknown as IncomingMessage & { headers: typeof headers }; + r.headers = headers; + return r; +} + +describe('ingestUpload', () => { + // Each test gets its own staging parent (via the IngestOptions root override) + // so assertions never read the shared global ~/.gitnexus/uploads, which other + // upload test files mutate concurrently under vitest's parallel forks. + let root: string; + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ingest-test-')); + }); + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }).catch(() => {}); + }); + + it('writes a manifest-described tree into the sandbox and returns its shape', async () => { + const { body, headers } = multipart([ + { name: 'manifest', value: JSON.stringify(['myrepo/a.js', 'myrepo/sub/b.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('alpha') }, + { name: 'files', filename: 'blob', data: Buffer.from('beta') }, + ]); + const result = await ingestUpload(mockReq(body, headers), undefined, { root }); + + expect(result.stageRoot.startsWith(await fs.realpath(root))).toBe(true); + expect(result.fileCount).toBe(2); + expect(result.topLevelName).toBe('myrepo'); + expect(result.totalBytes).toBe('alpha'.length + 'beta'.length); + expect(await fs.readFile(path.join(result.stageRoot, 'myrepo', 'a.js'), 'utf8')).toBe('alpha'); + expect(await fs.readFile(path.join(result.stageRoot, 'myrepo', 'sub', 'b.js'), 'utf8')).toBe( + 'beta', + ); + }); + + it('rejects a traversal path in the manifest and removes the staging dir', async () => { + const { body, headers } = multipart([ + { name: 'manifest', value: JSON.stringify(['../escape.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]); + await expect(ingestUpload(mockReq(body, headers), undefined, { root })).rejects.toMatchObject({ + status: 400, + }); + // The staging dir created by this call must not survive the rejection — + // the isolated root makes this exact (no concurrent test can add entries), + // and a missing/broken root throws rather than passing vacuously. + const entries = await fs.readdir(root); + expect(entries.filter((e) => e.startsWith(STAGING_PREFIX))).toEqual([]); + }); + + it('rejects a file part that arrives before the manifest', async () => { + const { body, headers } = multipart([ + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + { name: 'manifest', value: JSON.stringify(['a/x.js']) }, + ]); + await expect(ingestUpload(mockReq(body, headers), undefined, { root })).rejects.toThrow( + /Manifest must precede/, + ); + }); + + it('rejects when total bytes exceed the cap (413)', async () => { + const { body, headers } = multipart([ + { name: 'manifest', value: JSON.stringify(['r/big.bin']) }, + { name: 'files', filename: 'blob', data: Buffer.alloc(64, 1) }, + ]); + await expect( + ingestUpload(mockReq(body, headers), { maxTotalBytes: 16 }, { root }), + ).rejects.toMatchObject({ status: 413 }); + }); + + it('rejects an empty upload (0 files)', async () => { + const { body, headers } = multipart([{ name: 'manifest', value: JSON.stringify([]) }]); + await expect(ingestUpload(mockReq(body, headers), undefined, { root })).rejects.toThrow(); + }); + + it('exposes sane default caps', () => { + expect(DEFAULT_INGEST_LIMITS.maxTotalBytes).toBe(250 * 1024 * 1024); + expect(DEFAULT_INGEST_LIMITS.maxFiles).toBe(20000); + }); +}); diff --git a/gitnexus/test/unit/upload-sweep.test.ts b/gitnexus/test/unit/upload-sweep.test.ts new file mode 100644 index 000000000..a2667bbcb --- /dev/null +++ b/gitnexus/test/unit/upload-sweep.test.ts @@ -0,0 +1,62 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import { sweepStaleUploads } from '../../src/server/upload-sweep.js'; + +let root: string; + +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-sweep-test-')); +}); +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }).catch(() => {}); +}); + +describe('sweepStaleUploads', () => { + it('removes stale staging dirs but keeps recent ones and non-staging dirs', async () => { + await fs.mkdir(path.join(root, '.staging-old')); + await fs.mkdir(path.join(root, '.staging-new')); + await fs.mkdir(path.join(root, 'myrepo')); // a promoted (persistent) upload dir + + const now = 1_000_000_000_000; + // Age the "old" staging dir well past the threshold. + const old = new Date(now - 10 * 60 * 60 * 1000); + await fs.utimes(path.join(root, '.staging-old'), old, old); + const recent = new Date(now - 60 * 1000); + await fs.utimes(path.join(root, '.staging-new'), recent, recent); + + const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 }); + + expect(removed).toHaveLength(1); + expect(removed[0]).toContain('.staging-old'); + await expect(fs.access(path.join(root, '.staging-old'))).rejects.toBeTruthy(); + // Recent staging and the promoted repo dir survive. + await expect(fs.access(path.join(root, '.staging-new'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(root, 'myrepo'))).resolves.toBeUndefined(); + }); + + it('removes a stale promoted dir without a .gitnexus index, keeps one with it', async () => { + const now = 2_000_000_000_000; + const old = new Date(now - 10 * 60 * 60 * 1000); + + // Orphan: a failed analysis that never wrote an index. + await fs.mkdir(path.join(root, 'orphan')); + await fs.utimes(path.join(root, 'orphan'), old, old); + + // Registered: stale but carries the .gitnexus index → must be kept. + await fs.mkdir(path.join(root, 'registered', '.gitnexus'), { recursive: true }); + await fs.utimes(path.join(root, 'registered'), old, old); + + const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 }); + + expect(removed.some((r) => r.endsWith('orphan'))).toBe(true); + await expect(fs.access(path.join(root, 'orphan'))).rejects.toBeTruthy(); + await expect(fs.access(path.join(root, 'registered'))).resolves.toBeUndefined(); + }); + + it('tolerates a missing root', async () => { + const { removed } = await sweepStaleUploads({ root: path.join(root, 'does-not-exist') }); + expect(removed).toEqual([]); + }); +});