diff --git a/gitnexus-web/e2e/directory-picker.spec.ts b/gitnexus-web/e2e/directory-picker.spec.ts deleted file mode 100644 index c49f43be1..000000000 --- a/gitnexus-web/e2e/directory-picker.spec.ts +++ /dev/null @@ -1,325 +0,0 @@ -import { test, expect } from '@playwright/test'; - -/** - * E2E tests for the server-side directory picker (issue #1518). - * - * All tests mock the backend at the network level so they don't - * require a live gitnexus server. The /api/fs/list endpoint is - * intercepted to return controlled directory structures. - */ - -const BACKEND_URL = 'http://localhost:4747'; - -/** Standard backend mocks needed to reach the analyze form. */ -async function mockBackendForAnalyzeForm(page: import('@playwright/test').Page) { - await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] })); - await page.route(`${BACKEND_URL}/api/info`, (route) => - route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }), - ); - await page.route(`${BACKEND_URL}/api/heartbeat`, (route) => - route.fulfill({ - status: 200, - headers: { 'Content-Type': 'text/event-stream' }, - body: ':ok\n\n', - }), - ); -} - -/** Navigate to the Local Folder tab in the analyze form. */ -async function openLocalFolderTab(page: import('@playwright/test').Page) { - await page.goto('/'); - await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); - await page.getByRole('tab', { name: 'Local Folder' }).click(); -} - -// ── Directory picker: open and display ──────────────────────────────────── - -test.describe('Directory picker — open and display', () => { - test.beforeEach(async ({ page }) => { - await mockBackendForAnalyzeForm(page); - - await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => { - const url = new URL(route.request().url()); - const dir = url.searchParams.get('dir') ?? '/'; - - if (dir === '/') { - return route.fulfill({ - json: { entries: [{ name: 'workspace' }, { name: 'data' }, { name: 'home' }] }, - }); - } - if (dir === '/workspace') { - return route.fulfill({ - json: { entries: [{ name: 'my-project' }, { name: 'another-repo' }] }, - }); - } - if (dir === '/workspace/my-project') { - return route.fulfill({ json: { entries: [] } }); - } - return route.fulfill({ json: { entries: [] } }); - }); - }); - - test('clicking Browse opens the directory picker modal', async ({ page }, testInfo) => { - await openLocalFolderTab(page); - - await page.locator('[data-testid="browse-server-dirs"]').click(); - - await expect(page.locator('[data-testid="directory-picker-modal"]')).toBeVisible({ - timeout: 5_000, - }); - await page.screenshot({ path: testInfo.outputPath('picker-open.png') }); - }); - - test('picker shows root directories from /api/fs/list', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - await expect(page.locator('[data-testid="dir-entry-workspace"]')).toBeVisible({ - timeout: 5_000, - }); - await expect(page.locator('[data-testid="dir-entry-data"]')).toBeVisible(); - await expect(page.locator('[data-testid="dir-entry-home"]')).toBeVisible(); - }); - - test('current path shows / at root', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/'); - }); -}); - -// ── Directory picker: navigation ────────────────────────────────────────── - -test.describe('Directory picker — navigation', () => { - test.beforeEach(async ({ page }) => { - await mockBackendForAnalyzeForm(page); - - await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => { - const url = new URL(route.request().url()); - const dir = url.searchParams.get('dir') ?? '/'; - - if (dir === '/') { - return route.fulfill({ - json: { entries: [{ name: 'workspace' }, { name: 'data' }] }, - }); - } - if (dir === '/workspace') { - return route.fulfill({ - json: { entries: [{ name: 'my-project' }, { name: 'another-repo' }] }, - }); - } - if (dir === '/workspace/my-project') { - return route.fulfill({ json: { entries: [{ name: 'src' }] } }); - } - return route.fulfill({ json: { entries: [] } }); - }); - }); - - test('clicking a directory navigates into it', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - await page.locator('[data-testid="dir-entry-workspace"]').click(); - - await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({ - timeout: 5_000, - }); - await expect(page.locator('[data-testid="dir-entry-another-repo"]')).toBeVisible(); - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace'); - }); - - test('breadcrumb shows path segments after navigation', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - await page.locator('[data-testid="dir-entry-workspace"]').click(); - await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({ - timeout: 5_000, - }); - - // Breadcrumb should show "workspace" segment as a clickable button - const modal = page.locator('[data-testid="directory-picker-modal"]'); - await expect(modal.getByRole('button', { name: 'workspace', exact: true })).toBeVisible(); - }); - - test('clicking breadcrumb segment navigates back', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - // Navigate: / → /workspace → /workspace/my-project - await page.locator('[data-testid="dir-entry-workspace"]').click(); - await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({ - timeout: 5_000, - }); - await page.locator('[data-testid="dir-entry-my-project"]').click(); - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText( - '/workspace/my-project', - { timeout: 5_000 }, - ); - - // Click breadcrumb "workspace" to go back to /workspace - const modal = page.locator('[data-testid="directory-picker-modal"]'); - const breadcrumbSegments = modal.locator('button').filter({ hasText: 'workspace' }); - await breadcrumbSegments.first().click(); - - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace', { - timeout: 5_000, - }); - await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible(); - }); - - test('home button navigates to root', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - // Navigate into /workspace - await page.locator('[data-testid="dir-entry-workspace"]').click(); - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace', { - timeout: 5_000, - }); - - // Click home icon - await page.locator('[data-testid="directory-picker-home"]').click(); - - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/', { - timeout: 5_000, - }); - await expect(page.locator('[data-testid="dir-entry-workspace"]')).toBeVisible(); - }); -}); - -// ── Directory picker: selection ─────────────────────────────────────────── - -test.describe('Directory picker — selection', () => { - test.beforeEach(async ({ page }) => { - await mockBackendForAnalyzeForm(page); - - await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => { - const url = new URL(route.request().url()); - const dir = url.searchParams.get('dir') ?? '/'; - - if (dir === '/') { - return route.fulfill({ json: { entries: [{ name: 'workspace' }] } }); - } - if (dir === '/workspace') { - return route.fulfill({ json: { entries: [{ name: 'my-project' }] } }); - } - return route.fulfill({ json: { entries: [] } }); - }); - }); - - test('selecting a folder populates the local path input', async ({ page }, testInfo) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - // Navigate to /workspace - await page.locator('[data-testid="dir-entry-workspace"]').click(); - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace', { - timeout: 5_000, - }); - - // Click "Select this folder" - await page.locator('[data-testid="directory-picker-select"]').click(); - - // Modal should close - await expect(page.locator('[data-testid="directory-picker-modal"]')).not.toBeVisible(); - - // The local path input should contain the selected path - const pathInput = page.locator('input[type="text"]'); - await expect(pathInput).toHaveValue('/workspace'); - await page.screenshot({ path: testInfo.outputPath('path-populated.png') }); - }); - - test('selected path is an absolute path', async ({ page }) => { - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - await page.locator('[data-testid="dir-entry-workspace"]').click(); - await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({ - timeout: 5_000, - }); - await page.locator('[data-testid="dir-entry-my-project"]').click(); - await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText( - '/workspace/my-project', - { timeout: 5_000 }, - ); - - await page.locator('[data-testid="directory-picker-select"]').click(); - - const pathInput = page.locator('input[type="text"]'); - await expect(pathInput).toHaveValue('/workspace/my-project'); - }); - - test('closing the modal does not change the path input', async ({ page }) => { - await openLocalFolderTab(page); - - // Type a path manually first - const pathInput = page.locator('input[type="text"]'); - await pathInput.fill('/my/custom/path'); - - // Open and close the picker without selecting - await page.locator('[data-testid="browse-server-dirs"]').click(); - await expect(page.locator('[data-testid="directory-picker-modal"]')).toBeVisible({ - timeout: 5_000, - }); - - // Click the backdrop to close - await page - .locator('[data-testid="directory-picker-modal"]') - .locator('..') - .locator('div') - .first() - .click({ position: { x: 5, y: 5 }, force: true }); - - // Path input should keep the original value - await expect(pathInput).toHaveValue('/my/custom/path'); - }); -}); - -// ── Directory picker: edge cases ────────────────────────────────────────── - -test.describe('Directory picker — edge cases', () => { - test('shows empty state for a directory with no subdirectories', async ({ page }) => { - await mockBackendForAnalyzeForm(page); - await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => - route.fulfill({ json: { entries: [] } }), - ); - - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - const modal = page.locator('[data-testid="directory-picker-modal"]'); - await expect(modal.getByText('This directory is empty.')).toBeVisible({ timeout: 5_000 }); - }); - - test('shows error when /api/fs/list fails', async ({ page }) => { - await mockBackendForAnalyzeForm(page); - await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => - route.fulfill({ status: 500, json: { error: 'Internal server error' } }), - ); - - await openLocalFolderTab(page); - await page.locator('[data-testid="browse-server-dirs"]').click(); - - const modal = page.locator('[data-testid="directory-picker-modal"]'); - await expect(modal.locator('text=Go back')).toBeVisible({ timeout: 5_000 }); - }); - - test('manual path typing still works without opening picker', async ({ page }) => { - await mockBackendForAnalyzeForm(page); - - await openLocalFolderTab(page); - - // Type a path manually - const pathInput = page.locator('input[type="text"]'); - await pathInput.fill('/workspace/my-repo'); - - // The input should have the typed path - await expect(pathInput).toHaveValue('/workspace/my-repo'); - - // The Analyze button should be enabled - const analyzeBtn = page.getByRole('button', { name: /Analyze Repository/ }); - await expect(analyzeBtn).toBeEnabled(); - }); -}); diff --git a/gitnexus-web/e2e/folder-upload.spec.ts b/gitnexus-web/e2e/folder-upload.spec.ts new file mode 100644 index 000000000..c465547ab --- /dev/null +++ b/gitnexus-web/e2e/folder-upload.spec.ts @@ -0,0 +1,66 @@ +import { test, expect } from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * E2E for the browser folder-upload flow (replaces the removed server-side + * directory picker). Mocks the backend so no live gitnexus server is needed. + */ + +const BACKEND_URL = 'http://localhost:4747'; + +let fixtureDir: string; + +test.beforeAll(() => { + // A tiny "repo" folder; Playwright sets webkitRelativePath = /. + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-upload-e2e-')); + fixtureDir = path.join(root, 'myrepo'); + fs.mkdirSync(path.join(fixtureDir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(fixtureDir, 'README.md'), '# hi\n'); + fs.writeFileSync(path.join(fixtureDir, 'src', 'index.ts'), 'export const x = 1;\n'); +}); + +test.beforeEach(async ({ page }) => { + await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] })); + await page.route(`${BACKEND_URL}/api/info`, (route) => + route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }), + ); + await page.route(`${BACKEND_URL}/api/heartbeat`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: ':ok\n\n', + }), + ); +}); + +test('uploading a folder posts a multipart upload and starts analysis', async ({ page }) => { + let uploadContentType = ''; + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + uploadContentType = route.request().headers()['content-type'] ?? ''; + await route.fulfill({ json: { jobId: 'job-e2e', status: 'analyzing' } }); + }); + // SSE progress → immediately complete. + await page.route(`${BACKEND_URL}/api/analyze/job-e2e/progress`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }), + ); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); + + // Select the fixture folder via the hidden webkitdirectory input. + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + + // The upload endpoint should be hit with a multipart body, and the UI should + // leave the input phase (upload button no longer shown). + await expect.poll(() => uploadContentType).toContain('multipart/form-data'); + await expect(page.locator('[data-testid="upload-folder"]')).toBeHidden({ timeout: 10_000 }); +}); diff --git a/gitnexus-web/e2e/onboarding.spec.ts b/gitnexus-web/e2e/onboarding.spec.ts index 08ff7007b..147f55632 100644 --- a/gitnexus-web/e2e/onboarding.spec.ts +++ b/gitnexus-web/e2e/onboarding.spec.ts @@ -218,8 +218,8 @@ test.describe('Flow 3: Analyze form', () => { // Switch to Local Folder tab await page.getByRole('tab', { name: 'Local Folder' }).click(); - // Browse button should be visible (server-side directory picker) - await expect(page.locator('[data-testid="browse-server-dirs"]')).toBeVisible(); + // Upload-a-folder button should be visible (browser folder upload) + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); await page.screenshot({ path: testInfo.outputPath('local-folder-tab.png') }); }); diff --git a/gitnexus-web/src/components/DirectoryPicker.tsx b/gitnexus-web/src/components/DirectoryPicker.tsx deleted file mode 100644 index 07b8018c9..000000000 --- a/gitnexus-web/src/components/DirectoryPicker.tsx +++ /dev/null @@ -1,217 +0,0 @@ -import { useState, useEffect, useCallback, useRef } from 'react'; -import { Folder, ChevronRight, Home, X, Loader2, AlertCircle, Check } from '@/lib/lucide-icons'; -import { listDirectories, type DirEntry } from '../services/backend-client'; -import { useTranslation } from 'react-i18next'; - -interface DirectoryPickerProps { - open: boolean; - onClose: () => void; - onSelect: (absolutePath: string) => void; - initialDir?: string; -} - -export const DirectoryPicker = ({ open, onClose, onSelect, initialDir }: DirectoryPickerProps) => { - const { t } = useTranslation('onboarding'); - const [currentDir, setCurrentDir] = useState(initialDir ?? '/'); - const [entries, setEntries] = useState([]); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - const [isVisible, setIsVisible] = useState(false); - const panelRef = useRef(null); - - const fetchEntries = useCallback(async (dir: string) => { - setLoading(true); - setError(null); - try { - const result = await listDirectories(dir); - setEntries(result.entries); - setCurrentDir(dir); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : 'Failed to list directory'); - } finally { - setLoading(false); - } - }, []); - - useEffect(() => { - if (open) { - fetchEntries(initialDir ?? '/'); - requestAnimationFrame(() => setIsVisible(true)); - } else { - setIsVisible(false); - } - }, [open, initialDir, fetchEntries]); - - useEffect(() => { - if (!open) return; - const handleKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Escape') { - e.stopPropagation(); - onClose(); - } - }; - document.addEventListener('keydown', handleKeyDown); - return () => document.removeEventListener('keydown', handleKeyDown); - }, [open, onClose]); - - useEffect(() => { - if (open && panelRef.current) { - panelRef.current.focus(); - } - }, [open]); - - if (!open) return null; - - const segments = currentDir.split('/').filter(Boolean); - - const navigateTo = (dir: string) => { - fetchEntries(dir); - }; - - const handleBreadcrumbClick = (index: number) => { - const target = '/' + segments.slice(0, index + 1).join('/'); - navigateTo(target); - }; - - return ( -
- - ); -}; diff --git a/gitnexus-web/src/components/RepoAnalyzer.tsx b/gitnexus-web/src/components/RepoAnalyzer.tsx index 15981a992..33b46aeea 100644 --- a/gitnexus-web/src/components/RepoAnalyzer.tsx +++ b/gitnexus-web/src/components/RepoAnalyzer.tsx @@ -21,10 +21,11 @@ import { startAnalyze, cancelAnalyze, streamAnalyzeProgress, + uploadFolder, type JobProgress, } from '../services/backend-client'; import { AnalyzeProgress } from './AnalyzeProgress'; -import { DirectoryPicker } from './DirectoryPicker'; +import { filterRepoFiles } from '@/lib/upload-filter'; import { useTranslation } from 'react-i18next'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -167,7 +168,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const { t } = useTranslation(['common', 'errors', 'onboarding']); const inputId = useId(); const [mode, setMode] = useState('github'); - const [pickerOpen, setPickerOpen] = useState(false); + const [uploadPercent, setUploadPercent] = useState(null); + const [uploadSummary, setUploadSummary] = useState<{ count: number; dropped: number } | null>( + null, + ); const [githubUrl, setGithubUrl] = useState(''); const [gitlabUrl, setGitlabUrl] = useState(''); const [localPath, setLocalPath] = useState(''); @@ -183,6 +187,7 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const jobIdRef = useRef(null); const sseControllerRef = useRef(null); const completeTimerRef = useRef | null>(null); + const folderInputRef = useRef(null); useEffect(() => { return () => { @@ -237,8 +242,6 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp ? { url: gitlabUrl.trim() } : { path: localPath.trim() }; const { jobId } = await startAnalyze(request); - jobIdRef.current = jobId; - setPhase('analyzing'); const nameSource = mode === 'github' @@ -246,34 +249,67 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp : mode === 'gitlab' ? gitlabUrl.trim() : localPath.trim(); - const controller = streamAnalyzeProgress( - jobId, - (p) => setProgress(p), - (data) => { - const name = - data.repoName ?? - nameSource.split(/[/\\]/).filter(Boolean).at(-1) ?? - t('onboarding:repoAnalyzer.defaultRepoName'); - setCompletedRepoName(name); - setPhase('done'); - sseControllerRef.current = null; - completeTimerRef.current = setTimeout(() => { - completeTimerRef.current = null; - onComplete(name); - }, 1200); - }, - (errMsg) => { - setValidationError(errMsg || t('errors:analysisFailed')); - setPhase('error'); - }, - ); - sseControllerRef.current = controller; + trackJob(jobId, nameSource); } catch (err) { setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); setPhase('error'); } }; + // Drive an already-created analysis job through the SSE progress stream to + // completion. Shared by the path/URL analyze flow and the folder-upload flow. + const trackJob = (jobId: string, fallbackNameSource: string | null) => { + jobIdRef.current = jobId; + setPhase('analyzing'); + const controller = streamAnalyzeProgress( + jobId, + (p) => setProgress(p), + (data) => { + const name = + data.repoName ?? + (fallbackNameSource + ? fallbackNameSource.split(/[/\\]/).filter(Boolean).at(-1) + : undefined) ?? + t('onboarding:repoAnalyzer.defaultRepoName'); + setCompletedRepoName(name); + setPhase('done'); + sseControllerRef.current = null; + completeTimerRef.current = setTimeout(() => { + completeTimerRef.current = null; + onComplete(name); + }, 1200); + }, + (errMsg) => { + setValidationError(errMsg || t('errors:analysisFailed')); + setPhase('error'); + }, + ); + sseControllerRef.current = controller; + }; + + // Upload a browser-selected folder (webkitdirectory) and start analysis. The + // upload endpoint returns a jobId, which then joins the normal SSE flow. + const handleFolderUpload = async (fileList: FileList) => { + const { files, manifest, droppedCount } = filterRepoFiles(fileList); + if (files.length === 0) { + setValidationError(t('onboarding:repoAnalyzer.upload.empty')); + return; + } + setValidationError(null); + setUploadSummary({ count: files.length, dropped: droppedCount }); + setUploadPercent(0); + setPhase('starting'); + try { + const { jobId } = await uploadFolder(files, manifest, (pct) => setUploadPercent(pct)); + setUploadPercent(null); + trackJob(jobId, null); + } catch (err) { + setUploadPercent(null); + setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); + setPhase('error'); + } + }; + const handleCancel = async () => { sseControllerRef.current?.abort(); sseControllerRef.current = null; @@ -444,25 +480,54 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp )}
+ {/* Upload a folder from your computer — no server path or mount needed. + The browser can't expose an absolute path, so we upload the files. */} + { + if (e.target.files && e.target.files.length > 0) { + handleFolderUpload(e.target.files); + } + e.target.value = ''; + }} + /> - setPickerOpen(false)} - onSelect={(selectedPath) => { - setLocalPath(selectedPath); - setPickerOpen(false); - setValidationError(null); - }} - /> + {uploadPercent !== null && ( +
+
+
+
+

+ {t('onboarding:repoAnalyzer.upload.uploading', { percent: uploadPercent })} +

+
+ )} + {uploadSummary && uploadPercent === null && phase !== 'error' && ( +

+ {t('onboarding:repoAnalyzer.upload.selected', { + count: uploadSummary.count, + dropped: uploadSummary.dropped, + })} +

+ )}
)} diff --git a/gitnexus-web/src/lib/upload-filter.test.ts b/gitnexus-web/src/lib/upload-filter.test.ts new file mode 100644 index 000000000..e97b9ec2c --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import { filterRepoFiles, MAX_FILE_BYTES } from './upload-filter'; + +type FileLike = { name: string; size: number; webkitRelativePath?: string }; + +function f(webkitRelativePath: string, size = 10): FileLike { + const name = webkitRelativePath.split('/').pop() ?? webkitRelativePath; + return { name, size, webkitRelativePath }; +} + +describe('filterRepoFiles', () => { + it('keeps source files and builds an order-aligned manifest', () => { + const input = [f('repo/src/index.ts', 100), f('repo/README.md', 50)]; + const r = filterRepoFiles(input); + expect(r.files).toHaveLength(2); + expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/README.md']); + expect(r.totalBytes).toBe(150); + expect(r.droppedCount).toBe(0); + }); + + it('excludes .git / node_modules / build dirs anywhere in the path', () => { + const input = [ + f('repo/.git/HEAD'), + f('repo/node_modules/x/index.js'), + f('repo/dist/bundle.js'), + f('repo/src/app.ts'), + f('repo/.gitnexus/meta.json'), + ]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/src/app.ts']); + expect(r.droppedCount).toBe(4); + }); + + it('drops files over the per-file size cap', () => { + const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/small.ts']); + expect(r.droppedCount).toBe(1); + }); + + it('falls back to name when webkitRelativePath is absent', () => { + const r = filterRepoFiles([{ name: 'lone.ts', size: 5 }]); + expect(r.manifest).toEqual(['lone.ts']); + }); +}); diff --git a/gitnexus-web/src/lib/upload-filter.ts b/gitnexus-web/src/lib/upload-filter.ts new file mode 100644 index 000000000..a24520f74 --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.ts @@ -0,0 +1,73 @@ +/** + * Client-side pre-filter for a webkitdirectory folder upload. + * + * Drops VCS metadata, dependency/build directories, and oversized files before + * upload — `.git` alone is often larger than the working tree — so payloads + * stay small and the upload matches what the analyzer actually needs. Produces + * an order-aligned `manifest` of webkitRelativePaths (the server keys on this, + * not the multipart filename, which browsers rewrite). + */ + +/** Directory names excluded anywhere in a file's path. */ +export const EXCLUDED_DIRS = new Set([ + '.git', + '.hg', + '.svn', + 'node_modules', + 'vendor', + '.venv', + '__pycache__', + 'target', + 'dist', + 'build', + 'out', + '.next', + '.nuxt', + '.cache', + 'coverage', + '.idea', + '.gitnexus', +]); + +/** Per-file size cap; matches the server's per-file limit. */ +export const MAX_FILE_BYTES = 25 * 1024 * 1024; + +export interface FilterResult { + files: File[]; + manifest: string[]; + droppedCount: number; + totalBytes: number; +} + +type FileLike = Pick & { webkitRelativePath?: string }; + +/** + * Filter a webkitdirectory `FileList` (or array) into the files to upload plus + * their relative-path manifest. + */ +export function filterRepoFiles(input: ArrayLike): FilterResult { + const files: File[] = []; + const manifest: string[] = []; + let droppedCount = 0; + let totalBytes = 0; + + for (let i = 0; i < input.length; i++) { + const f = input[i]; + const rel = + f.webkitRelativePath && f.webkitRelativePath.length > 0 ? f.webkitRelativePath : f.name; + const segments = rel.split('/'); + if (segments.some((s) => EXCLUDED_DIRS.has(s))) { + droppedCount++; + continue; + } + if (f.size > MAX_FILE_BYTES) { + droppedCount++; + continue; + } + files.push(f as File); + manifest.push(rel); + totalBytes += f.size; + } + + return { files, manifest, droppedCount, totalBytes }; +} diff --git a/gitnexus-web/src/locales/en/onboarding.json b/gitnexus-web/src/locales/en/onboarding.json index 710311c65..9e1742d74 100644 --- a/gitnexus-web/src/locales/en/onboarding.json +++ b/gitnexus-web/src/locales/en/onboarding.json @@ -61,13 +61,12 @@ "gitlabRepositoryUrl": "GitLab Repository URL", "gitlabSupported": "Supports GitLab.com and self-hosted GitLab instances.", "localFolderPath": "Local Folder Path", - "browseForFolder": "Browse server directories", "hideBackground": "Hide (analysis continues in background)", - "directoryPicker": { - "title": "Browse Server Directories", - "select": "Select this folder", - "empty": "This directory is empty.", - "goBack": "Go back" + "upload": { + "button": "Upload a folder", + "uploading": "Uploading… {{percent}}%", + "selected": "{{count}} files ready ({{dropped}} skipped: .git, node_modules, build output)", + "empty": "No analyzable files found in that folder." } } } diff --git a/gitnexus-web/src/locales/zh-CN/onboarding.json b/gitnexus-web/src/locales/zh-CN/onboarding.json index 5ab5c8c9e..a4b16fbb4 100644 --- a/gitnexus-web/src/locales/zh-CN/onboarding.json +++ b/gitnexus-web/src/locales/zh-CN/onboarding.json @@ -61,13 +61,12 @@ "gitlabRepositoryUrl": "GitLab 仓库 URL", "gitlabSupported": "支持 GitLab.com 和自托管 GitLab 实例。", "localFolderPath": "本地文件夹路径", - "browseForFolder": "浏览服务器目录", "hideBackground": "隐藏（分析继续在后台进行）", - "directoryPicker": { - "title": "浏览服务器目录", - "select": "选择此文件夹", - "empty": "此目录为空。", - "goBack": "返回上级" + "upload": { + "button": "上传文件夹", + "uploading": "上传中… {{percent}}%", + "selected": "已准备 {{count}} 个文件（已跳过 {{dropped}} 个：.git、node_modules、构建产物）", + "empty": "该文件夹中未找到可分析的文件。" } } } diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index e4afa0b5b..f1a28be14 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -755,21 +755,56 @@ export const fetchClusterDetail = async (repo: string, name: string): Promise`) and start analysis. + * Sends the file blobs plus a JSON `manifest` of their relative paths — the + * multipart filename can't carry the path (browsers strip separators), so the + * manifest is the source of truth. Uses XHR for upload progress. Returns the + * analysis jobId, which the caller drives through the normal SSE flow. + */ +export const uploadFolder = ( + files: File[], + manifest: string[], + onProgress?: (percent: number) => void, +): Promise<{ jobId: string; status: string }> => { + return new Promise((resolve, reject) => { + const form = new FormData(); + // Manifest MUST precede the file parts (the server enforces this). + form.append('manifest', JSON.stringify(manifest)); + for (const f of files) form.append('files', f); -/** List subdirectories at the given absolute server-side path. */ -export const listDirectories = async (dir: string): Promise<{ entries: DirEntry[] }> => { - const response = await fetchWithTimeout( - `${_backendUrl}/api/fs/list?dir=${encodeURIComponent(dir)}`, - undefined, - 5_000, - ); - await assertOk(response); - return response.json() as Promise<{ entries: DirEntry[] }>; + const xhr = new XMLHttpRequest(); + xhr.open('POST', `${_backendUrl}/api/analyze/upload`); + xhr.timeout = 5 * 60_000; // up to 5 min for large repos + xhr.upload.onprogress = (e) => { + if (onProgress && e.lengthComputable) { + onProgress(Math.round((e.loaded / e.total) * 100)); + } + }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + try { + resolve(JSON.parse(xhr.responseText) as { jobId: string; status: string }); + } catch { + reject(new Error('Invalid server response')); + } + return; + } + let msg = `Upload failed (${xhr.status})`; + try { + const body = JSON.parse(xhr.responseText); + if (body?.error) msg = body.error; + } catch { + /* keep default message */ + } + reject(new Error(msg)); + }; + xhr.onerror = () => reject(new Error('Upload failed: network error')); + xhr.ontimeout = () => reject(new Error('Upload timed out')); + xhr.send(form); + }); }; // ── Analyze API ──────────────────────────────────────────────────────────── diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index e4fee45fc..3a49fda29 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -14,6 +14,7 @@ "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -50,6 +51,7 @@ "gitnexus": "dist/cli/index.js" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", @@ -1707,6 +1709,16 @@ "@types/node": "*" } }, + "node_modules/@types/busboy": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz", + "integrity": "sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2217,6 +2229,17 @@ "node": "18 || 20 || >=22" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -4819,6 +4842,14 @@ "dev": true, "license": "MIT" }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 293b651ed..16b54d598 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -58,6 +58,7 @@ "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -96,6 +97,7 @@ "tree-sitter-kotlin": "^0.3.8" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", diff --git a/gitnexus/src/server/analyze-upload.ts b/gitnexus/src/server/analyze-upload.ts new file mode 100644 index 000000000..ee58c58e8 --- /dev/null +++ b/gitnexus/src/server/analyze-upload.ts @@ -0,0 +1,156 @@ +/** + * POST /api/analyze/upload — analyze a browser folder upload. + * + * Securely ingests the multipart upload into a sandbox (upload-ingest.ts), + * promotes it to a persistent app-controlled directory, and analyzes it via + * the same job/worker machinery as a git clone — never returning a server + * path to the client. Factored as a dependency-injected handler so the job + * machinery (createJob + the worker launcher) can be mocked in unit tests. + */ + +import path from 'path'; +import fsp from 'fs/promises'; +import type { Request, Response } from 'express'; +import type { IncomingMessage } from 'http'; +import { ingestUpload } from './upload-ingest.js'; +import { UPLOAD_ROOT, getUploadDir, deriveUploadName } from './upload-paths.js'; +import { BadRequestError } from './validation.js'; + +export interface UploadJobRef { + id: string; + status: string; +} + +export interface AnalyzeUploadDeps { + /** Create (or throw on busy) an analysis job for the given upload dir. */ + createJob: (params: { repoPath: string }) => UploadJobRef; + /** Launch the analyze worker against an already-resolved repo directory. */ + launch: (job: UploadJobRef, targetPath: string, opts: { registryName: string }) => void; + /** Injectable for tests (defaults to the real ingestUpload). */ + ingest?: typeof ingestUpload; +} + +/** + * Find an available upload directory name, appending `-2`, `-3`, … on + * collision with an existing upload. Bounded to avoid an unbounded scan. + */ +async function pickAvailableName(base: string): Promise { + for (let i = 0; i < 100; i++) { + const name = i === 0 ? base : `${base}-${i + 1}`; + let dir: string; + try { + dir = getUploadDir(name); + } catch { + continue; + } + try { + await fsp.access(dir); + // exists → try the next suffix + } catch { + return name; // ENOENT → available + } + } + throw new BadRequestError('Could not allocate an upload directory', 409); +} + +export function createAnalyzeUploadHandler(deps: AnalyzeUploadDeps) { + const ingest = deps.ingest ?? ingestUpload; + + return async function handleAnalyzeUploadRequest(req: Request, res: Response): Promise { + let stageRoot: string | undefined; + let promotedDir: string | undefined; + let launched = false; + try { + const result = await ingest(req as unknown as IncomingMessage); + stageRoot = result.stageRoot; + + const baseName = deriveUploadName(result.topLevelName); + if (!baseName) { + throw new BadRequestError('Uploaded folder has no usable name'); + } + const finalName = await pickAvailableName(baseName); + const finalDir = getUploadDir(finalName); + + // createJob BEFORE promote: a busy server throws → 409 and nothing has + // been moved into place yet, so the staging dir is the only thing to clean. + let job: UploadJobRef; + try { + job = deps.createJob({ repoPath: finalDir }); + } catch (err) { + const msg = err instanceof Error ? err.message : ''; + if (msg.includes('already in progress')) { + throw new BadRequestError(msg, 409); + } + throw err; + } + + // webkitRelativePath prefixes every entry with the picked folder, so the + // real repo root is stageRoot/. Promote that inner dir. + const innerRoot = path.join(result.stageRoot, result.topLevelName); + try { + if (!(await fsp.stat(innerRoot)).isDirectory()) { + throw new BadRequestError('Upload must be a folder'); + } + } catch (err) { + if (err instanceof BadRequestError) throw err; + throw new BadRequestError('Upload must be a folder'); + } + + // Promote staging → persistent upload dir. Both live under UPLOAD_ROOT's + // filesystem, so this rename stays atomic (no EXDEV). + await fsp.mkdir(UPLOAD_ROOT, { recursive: true }); + await fsp.rename(innerRoot, finalDir); + promotedDir = finalDir; + const oldStage = stageRoot; + stageRoot = undefined; + await fsp.rm(oldStage, { recursive: true, force: true }).catch(() => {}); + + // Drop any crafted index the upload may have carried (a `.gitnexus` + // segment passes containment); the worker will build a fresh one. + await fsp + .rm(path.join(finalDir, '.gitnexus'), { recursive: true, force: true }) + .catch(() => {}); + + deps.launch(job, finalDir, { registryName: finalName }); + launched = true; + + res.status(202).json({ jobId: job.id, status: job.status }); + } catch (err) { + if (stageRoot) { + await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => {}); + } + if (promotedDir && !launched) { + await fsp.rm(promotedDir, { recursive: true, force: true }).catch(() => {}); + } + if (err instanceof BadRequestError) { + res.status(err.status).json({ error: err.message }); + return; + } + res.status(500).json({ error: 'Upload failed' }); + } + }; +} + +/** + * Per-route guard that restricts an endpoint to localhost browser origins. + * Non-browser requests (no Origin header, e.g. curl) pass through. This closes + * cross-origin reach (e.g. the allow-listed public deploy + Private Network + * Access) to write routes without affecting read routes. + */ +export function requireLocalhostOrigin(req: Request, res: Response, next: () => void): void { + const origin = req.headers.origin; + if (origin === undefined) { + next(); + return; + } + try { + const hostname = new URL(origin).hostname; + if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') { + next(); + return; + } + } catch { + /* malformed origin → reject */ + } + res.status(403).json({ error: 'This endpoint is restricted to localhost origins' }); +} diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 700b085eb..e25afe898 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -35,6 +35,9 @@ import { fileURLToPath, pathToFileURL } from 'url'; import { JobManager } from './analyze-job.js'; import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js'; import { extractRepoName, getCloneDir, cloneOrPull } from './git-clone.js'; +import { createAnalyzeUploadHandler, requireLocalhostOrigin } from './analyze-upload.js'; +import { UPLOAD_ROOT } from './upload-paths.js'; +import { sweepStaleUploads } from './upload-sweep.js'; import { logger, flushLoggerSync } from '../core/logger.js'; const _require = createRequire(import.meta.url); @@ -563,66 +566,6 @@ const requestedRepo = (req: express.Request): string | undefined => { return undefined; }; -/** - * Handle a GET /api/fs/list request. Lists subdirectories at a given - * absolute path. Extracted for unit testing (same rationale as - * handleFileRequest — avoids CodeQL js/missing-rate-limiting false - * positives in tests). - */ -export const handleFsListRequest = async ( - req: { query: any }, - res: { - status: (code: number) => { json: (body: any) => void }; - json: (body: any) => void; - }, -): Promise => { - try { - const raw = req.query.dir ?? '/'; - const dir = assertString(raw, 'dir'); - - if (!path.isAbsolute(dir)) { - res.status(400).json({ error: '"dir" must be an absolute path' }); - return; - } - // Skip the traversal guard for bare root directories (/ on Linux, C:\ on - // Windows) where path.normalize and path.resolve diverge on Windows. - const isRoot = dir === path.parse(dir).root; - if (!isRoot && path.normalize(dir) !== path.resolve(dir)) { - res.status(400).json({ error: '"dir" must not contain traversal sequences' }); - return; - } - - let dirents; - try { - dirents = await fs.readdir(dir, { withFileTypes: true }); - } catch (err: unknown) { - const code = (err as NodeJS.ErrnoException).code; - if (code === 'ENOENT' || code === 'ENOTDIR') { - res.status(404).json({ error: 'Directory not found' }); - return; - } - if (code === 'EACCES' || code === 'EPERM') { - res.status(403).json({ error: 'Permission denied' }); - return; - } - throw err; - } - - const entries = dirents - .filter((d) => d.isDirectory()) - .map((d) => ({ name: d.name })) - .sort((a, b) => a.name.localeCompare(b.name)); - - res.json({ entries }); - } catch (err: unknown) { - if (err instanceof BadRequestError) { - res.status(err.status).json({ error: err.message }); - return; - } - res.status(500).json({ error: 'Internal server error' }); - } -}; - /** * Handle a GET /api/file request body. Extracted from createServer's route * registration so it can be unit-tested without spinning up an HTTP server @@ -801,6 +744,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const cleanupMcp = mountMCPEndpoints(app, backend); const jobManager = new JobManager(); + // Backstop: remove any upload staging dirs orphaned by a previous crash. + void sweepStaleUploads().catch(() => {}); + // Shared repo lock — prevents concurrent analyze + embed on the same repo path, // which would corrupt LadybugDB (analyze calls closeLbug + initLbug while embed has queries in flight). const activeRepoPaths = new Set(); @@ -817,6 +763,149 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => activeRepoPaths.delete(repoPath); }; + // Launch the analyze worker for an already-resolved repo directory. Shared by + // the JSON /api/analyze route and the multipart /api/analyze/upload route so + // the lock + fork + auto-retry + IPC machinery lives in one place. + const launchAnalysisWorker = ( + job: { id: string }, + targetPath: string, + opts: { + force?: boolean; + embeddings?: boolean; + dropEmbeddings?: boolean; + registryName?: string; + }, + ): void => { + // Acquire shared repo lock (keyed on storagePath to match embed handler) + const analyzeLockKey = getStoragePath(targetPath); + const lockErr = acquireRepoLock(analyzeLockKey); + if (lockErr) { + jobManager.updateJob(job.id, { status: 'failed', error: lockErr }); + return; + } + + jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' }); + + // ── Worker fork with auto-retry ────────────────────────────── + const MAX_WORKER_RETRIES = 2; + const callerPath = fileURLToPath(import.meta.url); + const isDev = callerPath.endsWith('.ts'); + const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js'; + const workerPath = path.join(path.dirname(callerPath), workerFile); + const tsxHookArgs: string[] = isDev + ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] + : []; + + const forkWorker = () => { + const currentJob = jobManager.getJob(job.id); + if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed') return; + + const child = fork(workerPath, [], { + execArgv: [...tsxHookArgs, '--max-old-space-size=8192'], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }); + + // Capture stderr for crash diagnostics + let stderrChunks = ''; + child.stderr?.on('data', (chunk: Buffer) => { + stderrChunks += chunk.toString(); + if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096); + }); + + child.on('message', (msg: any) => { + if (msg.type === 'progress') { + jobManager.updateJob(job.id, { + status: 'analyzing', + progress: { phase: msg.phase, percent: msg.percent, message: msg.message }, + }); + } else if (msg.type === 'complete') { + releaseRepoLock(analyzeLockKey); + // Reinitialize backend BEFORE marking complete — ensures the new + // repo is queryable when the client receives the SSE complete event. + backend + .init() + .then(() => { + jobManager.updateJob(job.id, { + status: 'complete', + repoName: msg.result.repoName, + }); + }) + .catch((err) => { + logger.error({ err }, 'backend.init() failed after analyze:'); + jobManager.updateJob(job.id, { + status: 'failed', + error: 'Server failed to reload after analysis. Try again.', + }); + }); + } else if (msg.type === 'error') { + releaseRepoLock(analyzeLockKey); + jobManager.updateJob(job.id, { + status: 'failed', + error: msg.message, + }); + } + }); + + child.on('error', (err) => { + releaseRepoLock(analyzeLockKey); + jobManager.updateJob(job.id, { + status: 'failed', + error: `Worker process error: ${err.message}`, + }); + }); + + child.on('exit', (code) => { + const j = jobManager.getJob(job.id); + if (!j || j.status === 'complete' || j.status === 'failed') return; + + // Worker crashed — attempt retry if under the limit + if (j.retryCount < MAX_WORKER_RETRIES) { + j.retryCount++; + const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s + const lastErr = stderrChunks.trim().split('\n').pop() || ''; + logger.warn( + `Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` + + (lastErr ? `: ${lastErr}` : ''), + ); + jobManager.updateJob(job.id, { + status: 'analyzing', + progress: { + phase: 'retrying', + percent: j.progress.percent, + message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`, + }, + }); + stderrChunks = ''; + setTimeout(forkWorker, delay); + } else { + // Exhausted retries — permanent failure + releaseRepoLock(analyzeLockKey); + jobManager.updateJob(job.id, { + status: 'failed', + error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`, + }); + } + }); + + // Register child for cancellation + timeout tracking + jobManager.registerChild(job.id, child); + + // Send start command to child + child.send({ + type: 'start', + repoPath: targetPath, + options: { + force: !!opts.force, + embeddings: !!opts.embeddings, + dropEmbeddings: !!opts.dropEmbeddings, + ...(opts.registryName ? { registryName: opts.registryName } : {}), + }, + }); + }; + + forkWorker(); + }; + /** * Maximum time the hold-queue will wait for an active analysis job to complete. * Must stay in sync with the frontend's `fetchRepoInfo({ awaitAnalysis: true })` timeout. @@ -1054,6 +1143,14 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => } } + // 2b. Delete the uploaded repo dir if entry.path lives under + // UPLOAD_ROOT. Drive this off entry.path (not a name-rederived dir) so + // a same-named clone is never affected. + const resolvedEntry = path.resolve(entry.path); + if (resolvedEntry === UPLOAD_ROOT || resolvedEntry.startsWith(UPLOAD_ROOT + path.sep)) { + await fs.rm(resolvedEntry, { recursive: true, force: true }).catch(() => {}); + } + // 3. Unregister from the global registry const { unregisterRepo } = await import('../storage/repo-manager.js'); await unregisterRepo(entry.path); @@ -1325,8 +1422,6 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => await handleFileRequest(req, res, entry.path); }); - app.get('/api/fs/list', createRouteLimiter(), (req, res) => handleFsListRequest(req, res)); - // Grep — regex search across file contents in the indexed repo // Uses filesystem-based search for memory efficiency (never loads all files into memory) // Rate-limited (CodeQL js/missing-rate-limiting): scans every file in @@ -1490,229 +1585,127 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // ── Analyze API ────────────────────────────────────────────────────── // POST /api/analyze — start a new analysis job - app.post('/api/analyze', createRouteLimiter({ limit: 10 }), async (req, res) => { - try { - const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings } = req.body; + app.post( + '/api/analyze', + createRouteLimiter({ limit: 10 }), + requireLocalhostOrigin, + async (req, res) => { + try { + const { url: repoUrl, force, embeddings, dropEmbeddings } = req.body; + // `path` is canonicalized below (realpath), so keep it mutable. + let repoLocalPath: string | undefined = req.body.path; - // Input type validation - if (repoUrl !== undefined && typeof repoUrl !== 'string') { - res.status(400).json({ error: '"url" must be a string' }); - return; - } - if (repoLocalPath !== undefined && typeof repoLocalPath !== 'string') { - res.status(400).json({ error: '"path" must be a string' }); - return; - } - - if (!repoUrl && !repoLocalPath) { - res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' }); - return; - } - - // Path validation: require absolute path, reject traversal (e.g. /tmp/../etc/passwd) - if (repoLocalPath) { - if (!path.isAbsolute(repoLocalPath)) { - res.status(400).json({ error: '"path" must be an absolute path' }); + // Input type validation + if (repoUrl !== undefined && typeof repoUrl !== 'string') { + res.status(400).json({ error: '"url" must be a string' }); return; } - if (path.normalize(repoLocalPath) !== path.resolve(repoLocalPath)) { - res.status(400).json({ error: '"path" must not contain traversal sequences' }); + if (repoLocalPath !== undefined && typeof repoLocalPath !== 'string') { + res.status(400).json({ error: '"path" must be a string' }); return; } - } - const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath }); + if (!repoUrl && !repoLocalPath) { + res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' }); + return; + } - // If job was already running (dedup), just return its id - if (job.status !== 'queued') { - res.status(202).json({ jobId: job.id, status: job.status }); - return; - } - - // Mark as active synchronously to prevent race with concurrent requests - jobManager.updateJob(job.id, { status: 'cloning' }); - - // Start async work — don't await - (async () => { - let targetPath = repoLocalPath; - try { - // Clone if URL provided - if (repoUrl && !repoLocalPath) { - const repoName = extractRepoName(repoUrl); - targetPath = getCloneDir(repoName); - - jobManager.updateJob(job.id, { - status: 'cloning', - repoName, - progress: { phase: 'cloning', percent: 0, message: `Cloning ${repoUrl}...` }, - }); - - await cloneOrPull(repoUrl, targetPath, (progress) => { - jobManager.updateJob(job.id, { - progress: { phase: progress.phase, percent: 5, message: progress.message }, - }); - }); - } - - if (!targetPath) { - throw new Error('No target path resolved'); - } - - // Acquire shared repo lock (keyed on storagePath to match embed handler) - const analyzeLockKey = getStoragePath(targetPath); - const lockErr = acquireRepoLock(analyzeLockKey); - if (lockErr) { - jobManager.updateJob(job.id, { status: 'failed', error: lockErr }); + // Path validation. The previous `normalize !== resolve` guard was inert + // (both collapse `..` identically) and only false-rejected trailing + // slashes. Analyzing a local path the operator names is the tool's + // intended capability (same as the CLI); the cross-origin reach is what + // mattered, and that is closed by requireLocalhostOrigin above. Here we + // just require an absolute path that actually exists and is a directory, + // using its realpath-canonical form downstream. + if (repoLocalPath) { + if (!path.isAbsolute(repoLocalPath)) { + res.status(400).json({ error: '"path" must be an absolute path' }); return; } - - jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' }); - - // ── Worker fork with auto-retry ────────────────────────────── - // - // Forks a child process with 8GB heap. If the worker crashes - // (OOM, native addon segfault, etc.), it retries up to - // MAX_WORKER_RETRIES times with exponential backoff before - // marking the job as permanently failed. - // - // In dev mode (tsx), registers the tsx ESM hook via a file:// - // URL so the child can compile TypeScript on-the-fly. - - const MAX_WORKER_RETRIES = 2; - const callerPath = fileURLToPath(import.meta.url); - const isDev = callerPath.endsWith('.ts'); - const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js'; - const workerPath = path.join(path.dirname(callerPath), workerFile); - const tsxHookArgs: string[] = isDev - ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] - : []; - - const forkWorker = () => { - const currentJob = jobManager.getJob(job.id); - if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed') + try { + const resolved = await fs.realpath(repoLocalPath); + if (!(await fs.stat(resolved)).isDirectory()) { + res.status(400).json({ error: '"path" must be a directory' }); return; - - const child = fork(workerPath, [], { - execArgv: [...tsxHookArgs, '--max-old-space-size=8192'], - stdio: ['ignore', 'pipe', 'pipe', 'ipc'], - }); - - // Capture stderr for crash diagnostics - let stderrChunks = ''; - child.stderr?.on('data', (chunk: Buffer) => { - stderrChunks += chunk.toString(); - if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096); - }); - - child.on('message', (msg: any) => { - if (msg.type === 'progress') { - jobManager.updateJob(job.id, { - status: 'analyzing', - progress: { phase: msg.phase, percent: msg.percent, message: msg.message }, - }); - } else if (msg.type === 'complete') { - releaseRepoLock(analyzeLockKey); - // Reinitialize backend BEFORE marking complete — ensures the new - // repo is queryable when the client receives the SSE complete event. - backend - .init() - .then(() => { - jobManager.updateJob(job.id, { - status: 'complete', - repoName: msg.result.repoName, - }); - }) - .catch((err) => { - logger.error({ err }, 'backend.init() failed after analyze:'); - jobManager.updateJob(job.id, { - status: 'failed', - error: 'Server failed to reload after analysis. Try again.', - }); - }); - } else if (msg.type === 'error') { - releaseRepoLock(analyzeLockKey); - jobManager.updateJob(job.id, { - status: 'failed', - error: msg.message, - }); - } - }); - - child.on('error', (err) => { - releaseRepoLock(analyzeLockKey); - jobManager.updateJob(job.id, { - status: 'failed', - error: `Worker process error: ${err.message}`, - }); - }); - - child.on('exit', (code) => { - const j = jobManager.getJob(job.id); - if (!j || j.status === 'complete' || j.status === 'failed') return; - - // Worker crashed — attempt retry if under the limit - if (j.retryCount < MAX_WORKER_RETRIES) { - j.retryCount++; - const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s - const lastErr = stderrChunks.trim().split('\n').pop() || ''; - logger.warn( - `Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` + - (lastErr ? `: ${lastErr}` : ''), - ); - jobManager.updateJob(job.id, { - status: 'analyzing', - progress: { - phase: 'retrying', - percent: j.progress.percent, - message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`, - }, - }); - stderrChunks = ''; - setTimeout(forkWorker, delay); - } else { - // Exhausted retries — permanent failure - releaseRepoLock(analyzeLockKey); - jobManager.updateJob(job.id, { - status: 'failed', - error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`, - }); - } - }); - - // Register child for cancellation + timeout tracking - jobManager.registerChild(job.id, child); - - // Send start command to child - child.send({ - type: 'start', - repoPath: targetPath, - options: { - force: !!force, - embeddings: !!embeddings, - dropEmbeddings: !!dropEmbeddings, - }, - }); - }; - - forkWorker(); - } catch (err: any) { - if (targetPath) releaseRepoLock(getStoragePath(targetPath)); - jobManager.updateJob(job.id, { - status: 'failed', - error: err.message || 'Analysis failed', - }); + } + repoLocalPath = resolved; + } catch { + res.status(404).json({ error: '"path" does not exist or is not accessible' }); + return; + } } - })(); - res.status(202).json({ jobId: job.id, status: job.status }); - } catch (err: any) { - if (err.message?.includes('already in progress')) { - res.status(409).json({ error: err.message }); - } else { - res.status(500).json({ error: err.message || 'Failed to start analysis' }); + const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath }); + + // If job was already running (dedup), just return its id + if (job.status !== 'queued') { + res.status(202).json({ jobId: job.id, status: job.status }); + return; + } + + // Mark as active synchronously to prevent race with concurrent requests + jobManager.updateJob(job.id, { status: 'cloning' }); + + // Start async work — don't await + (async () => { + let targetPath = repoLocalPath; + try { + // Clone if URL provided + if (repoUrl && !repoLocalPath) { + const repoName = extractRepoName(repoUrl); + targetPath = getCloneDir(repoName); + + jobManager.updateJob(job.id, { + status: 'cloning', + repoName, + progress: { phase: 'cloning', percent: 0, message: `Cloning ${repoUrl}...` }, + }); + + await cloneOrPull(repoUrl, targetPath, (progress) => { + jobManager.updateJob(job.id, { + progress: { phase: progress.phase, percent: 5, message: progress.message }, + }); + }); + } + + if (!targetPath) { + throw new Error('No target path resolved'); + } + + launchAnalysisWorker(job, targetPath, { force, embeddings, dropEmbeddings }); + } catch (err: any) { + if (targetPath) releaseRepoLock(getStoragePath(targetPath)); + jobManager.updateJob(job.id, { + status: 'failed', + error: err.message || 'Analysis failed', + }); + } + })(); + + res.status(202).json({ jobId: job.id, status: job.status }); + } catch (err: any) { + if (err.message?.includes('already in progress')) { + res.status(409).json({ error: err.message }); + } else { + res.status(500).json({ error: err.message || 'Failed to start analysis' }); + } } - } - }); + }, + ); + + // POST /api/analyze/upload — analyze a browser folder upload. + // Securely ingests the multipart upload into a sandbox, promotes it to a + // persistent dir, and analyzes it via the shared job/worker machinery. + // localhost-only (no cross-origin write reach) + conservative rate limit. + app.post( + '/api/analyze/upload', + createRouteLimiter({ limit: 5 }), + requireLocalhostOrigin, + createAnalyzeUploadHandler({ + createJob: (params) => jobManager.createJob(params), + launch: (job, targetPath, opts) => launchAnalysisWorker(job, targetPath, opts), + }), + ); // GET /api/analyze/:jobId — poll job status app.get('/api/analyze/:jobId', (req, res) => { diff --git a/gitnexus/src/server/upload-ingest.ts b/gitnexus/src/server/upload-ingest.ts new file mode 100644 index 000000000..b88f31c91 --- /dev/null +++ b/gitnexus/src/server/upload-ingest.ts @@ -0,0 +1,304 @@ +/** + * Secure ingestion of a browser folder upload (multipart/form-data). + * + * Replaces the path-injection-prone GET /api/fs/list directory listing. The + * client streams the selected files plus a JSON `manifest` of their + * webkitRelativePaths; we write each into an mkdtemp staging dir under + * UPLOAD_ROOT with PROVABLE containment (resolve-then-contain), hard resource + * caps, and guaranteed cleanup on every failure/abort path. No client value + * ever reaches a filesystem READ — the server only writes into a sandbox it + * created, then hands that sandbox to the analysis pipeline. + * + * Security references: CodeQL js/path-injection (resolve + startsWith(root+sep)), + * OWASP File Upload / Path Traversal. + */ + +import path from 'path'; +import fs from 'fs'; +import fsp from 'fs/promises'; +import type { IncomingMessage } from 'http'; +import busboy from 'busboy'; +import { UPLOAD_ROOT, STAGING_PREFIX } from './upload-paths.js'; +import { BadRequestError } from './validation.js'; + +export interface IngestLimits { + /** Aggregate bytes across all files (busboy has no aggregate limit). */ + maxTotalBytes: number; + /** Per-file byte cap. */ + maxFileBytes: number; + /** Maximum number of files. */ + maxFiles: number; + /** Maximum multipart parts (files + fields). */ + maxParts: number; + /** Maximum directories created (inode-exhaustion guard). */ + maxDirs: number; + /** Maximum size of the manifest field. */ + maxFieldBytes: number; +} + +export const DEFAULT_INGEST_LIMITS: IngestLimits = { + maxTotalBytes: 250 * 1024 * 1024, + maxFileBytes: 25 * 1024 * 1024, + maxFiles: 20000, + maxParts: 20100, + maxDirs: 50000, + maxFieldBytes: 2 * 1024 * 1024, +}; + +const MAX_PATH_DEPTH = 64; +const MAX_PATH_LENGTH = 4096; + +export interface IngestResult { + /** Absolute path to the populated staging directory (realpath-canonical). */ + stageRoot: string; + fileCount: number; + totalBytes: number; + /** First path segment shared by the uploaded tree (the picked folder). */ + topLevelName: string; +} + +/** + * Resolve a client-provided relative path to an absolute destination PROVABLY + * contained within `stageRoot`. Throws BadRequestError on any unsafe input. + * This is the load-bearing path-traversal-on-write control; keep it pure and + * unit-tested. + */ +export function resolveContainedDest(stageRoot: string, rel: unknown): string { + if (typeof rel !== 'string' || rel.length === 0) { + throw new BadRequestError('Invalid upload path'); + } + if (rel.length > MAX_PATH_LENGTH) { + throw new BadRequestError('Upload path too long'); + } + // webkitRelativePath is always relative; a leading slash is absolute/hostile. + if (rel.startsWith('/')) { + throw new BadRequestError('Invalid upload path'); + } + // Browsers emit forward slashes only; a NUL byte or backslash is hostile. + if (rel.includes('\u0000') || rel.includes('\\')) { + throw new BadRequestError('Invalid upload path'); + } + const rawSegments = rel.split('/').filter((s) => s.length > 0); + if (rawSegments.length === 0 || rawSegments.length > MAX_PATH_DEPTH) { + throw new BadRequestError('Invalid upload path'); + } + const segments: string[] = []; + for (const seg of rawSegments) { + // Normalize so NFC/NFD variants don't collide silently on case/unicode + // -folding filesystems (macOS/Windows). + const s = seg.normalize('NFC'); + if (s === '.' || s === '..') { + throw new BadRequestError('Upload path must not contain traversal segments'); + } + segments.push(s); + } + const dest = path.resolve(stageRoot, segments.join(path.sep)); + // Suffix path.sep so a sibling prefix (/sandbox-evil vs /sandbox) can't pass. + if (dest !== stageRoot && !dest.startsWith(stageRoot + path.sep)) { + throw new BadRequestError('Upload path escapes the sandbox'); + } + return dest; +} + +interface DirState { + dirCount: number; + limits: IngestLimits; +} + +/** + * Create the parent directories of `destFile` one segment at a time, asserting + * after each `mkdir` that the segment is a real directory (not a symlink + * swapped in mid-stream) still inside `stageRoot`. Counts created dirs against + * the inode-exhaustion cap. + */ +function mkdirContained(stageRoot: string, destFile: string, state: DirState): void { + const parent = path.dirname(destFile); + const relParent = path.relative(stageRoot, parent); + if (relParent === '' || relParent === '.') return; + const segs = relParent.split(path.sep).filter(Boolean); + let cur = stageRoot; + for (const seg of segs) { + cur = path.join(cur, seg); + let made = false; + try { + fs.mkdirSync(cur); + made = true; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + } + const st = fs.lstatSync(cur); + if (st.isSymbolicLink() || !st.isDirectory()) { + throw new BadRequestError('Upload path escapes the sandbox'); + } + if (made) { + state.dirCount++; + if (state.dirCount > state.limits.maxDirs) { + throw new BadRequestError('Too many directories in upload', 413); + } + } + } +} + +/** + * Parse and securely write a multipart folder upload into a fresh staging + * directory under UPLOAD_ROOT. Resolves with the populated staging dir, or + * rejects with a BadRequestError (status 400/413) after removing the staging + * dir. The caller owns promotion + cleanup of the returned `stageRoot`. + */ +export async function ingestUpload( + req: IncomingMessage, + limitsOverride?: Partial, +): Promise { + const limits = { ...DEFAULT_INGEST_LIMITS, ...limitsOverride }; + await fsp.mkdir(UPLOAD_ROOT, { recursive: true }); + // mkdtemp creates the dir mode 0o700 (owner-only); realpath canonicalizes + // the root so the containment prefix check is exact. + const stageRoot = await fsp.realpath(await fsp.mkdtemp(path.join(UPLOAD_ROOT, STAGING_PREFIX))); + + let cleaned = false; + const cleanup = async (): Promise => { + if (cleaned) return; + cleaned = true; + await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => {}); + }; + + return new Promise((resolve, reject) => { + let settled = false; + let manifest: string[] | null = null; + let fileIndex = 0; + let fileCount = 0; + let totalBytes = 0; + let topLevelName = ''; + const dirState: DirState = { dirCount: 0, limits }; + const writePromises: Promise[] = []; + + const bb = busboy({ + headers: req.headers, + limits: { + fileSize: limits.maxFileBytes, + files: limits.maxFiles, + parts: limits.maxParts, + fields: 10, + fieldNameSize: 200, + fieldSize: limits.maxFieldBytes, + headerPairs: 2000, + }, + }); + + const fail = (err: Error): void => { + if (settled) return; + settled = true; + try { + req.unpipe(bb); + } catch { + /* ignore */ + } + try { + req.resume(); // drain remaining body so the socket isn't left hanging + } catch { + /* ignore */ + } + void cleanup().finally(() => reject(err)); + }; + + bb.on('field', (name: string, val: string) => { + if (name !== 'manifest') return; + try { + const parsed = JSON.parse(val); + if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === 'string')) { + return fail(new BadRequestError('Invalid manifest')); + } + manifest = parsed as string[]; + } catch { + fail(new BadRequestError('Invalid manifest')); + } + }); + + bb.on('file', (_name: string, stream: NodeJS.ReadableStream, _info: unknown) => { + if (settled) { + stream.resume(); + return; + } + if (manifest === null) { + // The manifest field MUST arrive before any file part. + stream.resume(); + return fail(new BadRequestError('Manifest must precede file parts')); + } + const idx = fileIndex++; + const rel = manifest[idx]; + let dest: string; + try { + dest = resolveContainedDest(stageRoot, rel); + if (!topLevelName) { + // The on-disk top folder uses the NFC-normalized first segment + // (matching how each path segment is written); the promote step + // renames stageRoot/ into place. + topLevelName = (String(rel).split('/').filter(Boolean)[0] ?? '').normalize('NFC'); + } + mkdirContained(stageRoot, dest, dirState); + } catch (err) { + stream.resume(); + return fail(err as Error); + } + fileCount++; + const ws = fs.createWriteStream(dest, { flags: 'wx' }); + const p = new Promise((res, rej) => { + stream.on('data', (chunk: Buffer) => { + totalBytes += chunk.length; + if (totalBytes > limits.maxTotalBytes) { + stream.unpipe(ws); + ws.destroy(); + rej(new BadRequestError('Upload exceeds total size limit', 413)); + } + }); + stream.on('limit', () => { + ws.destroy(); + rej(new BadRequestError('File exceeds size limit', 413)); + }); + stream.on('error', rej); + ws.on('error', rej); + ws.on('finish', () => res()); + stream.pipe(ws); + }); + writePromises.push(p); + p.catch(fail); + }); + + bb.on('filesLimit', () => fail(new BadRequestError('Too many files in upload', 413))); + bb.on('partsLimit', () => fail(new BadRequestError('Too many parts in upload', 413))); + bb.on('fieldsLimit', () => fail(new BadRequestError('Too many fields in upload'))); + bb.on('error', (err: unknown) => + fail(err instanceof Error ? err : new BadRequestError('Upload parse error')), + ); + + bb.on('close', () => { + if (settled) return; + Promise.all(writePromises) + .then(() => { + if (settled) return; + if (manifest === null) return fail(new BadRequestError('Missing manifest')); + if (fileCount === 0) return fail(new BadRequestError('Empty upload')); + if (fileCount !== manifest.length) { + return fail(new BadRequestError('Manifest/file count mismatch')); + } + if (!topLevelName) { + return fail(new BadRequestError('Could not determine upload folder name')); + } + settled = true; + resolve({ stageRoot, fileCount, totalBytes, topLevelName }); + }) + .catch(() => { + /* a write rejected → fail() already invoked via p.catch */ + }); + }); + + req.on('aborted', () => { + if (!settled) fail(new BadRequestError('Upload aborted')); + }); + req.on('error', (err: unknown) => + fail(err instanceof Error ? err : new BadRequestError('Request error')), + ); + + req.pipe(bb); + }); +} diff --git a/gitnexus/src/server/upload-paths.ts b/gitnexus/src/server/upload-paths.ts new file mode 100644 index 000000000..cae4b6d8e --- /dev/null +++ b/gitnexus/src/server/upload-paths.ts @@ -0,0 +1,62 @@ +/** + * Upload working-directory paths. + * + * Browser folder uploads are written into ~/.gitnexus/uploads/{name}/ — a + * sibling of the clone root (git-clone.ts CLONE_ROOT) — so an uploaded repo + * persists and behaves like a cloned one (the graph UI's /api/file reads its + * files after analysis, and DELETE /api/repo removes it). Staging happens in + * an mkdtemp dir *under* UPLOAD_ROOT so the promote rename stays on one + * filesystem and remains atomic (a rename from os.tmpdir() could trip EXDEV — + * the exact Docker case this feature targets; see bridge-db.ts for the same + * anchored-staging pattern). + */ + +import path from 'path'; +import os from 'os'; +import { sanitizeRepoName } from '../storage/git.js'; + +/** Root directory for all uploaded repositories. Targets must resolve inside this. */ +export const UPLOAD_ROOT = path.resolve(path.join(os.homedir(), '.gitnexus', 'uploads')); + +/** Prefix for per-upload staging directories created under UPLOAD_ROOT. */ +export const STAGING_PREFIX = '.staging-'; + +// Filesystem-safe repo name: alphanumerics plus `. _ -`. Mirrors +// git-clone.ts REPO_NAME_PATTERN so getUploadDir(name) cannot escape +// UPLOAD_ROOT regardless of how the caller derived the name. +const UPLOAD_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; + +/** + * Get the upload target directory for a repo name. + * + * Re-validates at the boundary (callers may derive the name from an untrusted + * manifest). Rejects `.`, `..`, the `'unknown'` sentinel that sanitizeRepoName + * emits for un-nameable inputs, names beginning with `.` (which would collide + * with the `.staging-` prefix), and anything outside the safe charset. + */ +export function getUploadDir(repoName: string): string { + if ( + !repoName || + repoName === '.' || + repoName === '..' || + repoName === 'unknown' || + repoName.startsWith('.') || + !UPLOAD_NAME_PATTERN.test(repoName) + ) { + throw new Error('Invalid repository name'); + } + return path.join(UPLOAD_ROOT, repoName); +} + +/** + * Derive a filesystem-safe upload directory name from the manifest's + * top-level folder. Returns null when the name is un-nameable (so the caller + * rejects with 400 rather than colliding everyone on `UPLOAD_ROOT/unknown`). + */ +export function deriveUploadName(topLevelName: string): string | null { + const safe = sanitizeRepoName(topLevelName); + if (safe === 'unknown' || safe === '.' || safe === '..' || safe.startsWith('.')) { + return null; + } + return safe; +} diff --git a/gitnexus/src/server/upload-sweep.ts b/gitnexus/src/server/upload-sweep.ts new file mode 100644 index 000000000..3f2747dc7 --- /dev/null +++ b/gitnexus/src/server/upload-sweep.ts @@ -0,0 +1,52 @@ +/** + * Backstop cleanup for abandoned upload staging directories. + * + * A crashed/killed process can leave a `.staging-*` directory under + * UPLOAD_ROOT (the normal path removes it on success/failure/abort). This + * sweep, run once at server startup, removes staging dirs older than a + * threshold. Promoted upload dirs are persistent registered repos (like + * clones) and are NOT touched here — they are removed via DELETE /api/repo. + */ + +import path from 'path'; +import fsp from 'fs/promises'; +import { UPLOAD_ROOT, STAGING_PREFIX } from './upload-paths.js'; + +export interface SweepOptions { + /** Remove staging dirs older than this (default 6h). */ + maxAgeMs?: number; + /** Override the root to sweep (defaults to UPLOAD_ROOT; for tests). */ + root?: string; + /** Clock injection for tests. */ + now?: number; +} + +export async function sweepStaleUploads(opts: SweepOptions = {}): Promise<{ removed: string[] }> { + const maxAgeMs = opts.maxAgeMs ?? 6 * 60 * 60 * 1000; + const root = opts.root ?? UPLOAD_ROOT; + const now = opts.now ?? Date.now(); + const removed: string[] = []; + + let entries; + try { + entries = await fsp.readdir(root, { withFileTypes: true }); + } catch { + return { removed }; // root does not exist yet — nothing to sweep + } + + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.startsWith(STAGING_PREFIX)) continue; + const full = path.join(root, entry.name); + try { + const st = await fsp.stat(full); + if (now - st.mtimeMs > maxAgeMs) { + await fsp.rm(full, { recursive: true, force: true }).catch(() => {}); + removed.push(full); + } + } catch { + /* stat race — skip */ + } + } + + return { removed }; +} diff --git a/gitnexus/test/unit/api-analyze-upload.test.ts b/gitnexus/test/unit/api-analyze-upload.test.ts new file mode 100644 index 000000000..35786b4dd --- /dev/null +++ b/gitnexus/test/unit/api-analyze-upload.test.ts @@ -0,0 +1,221 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { Readable } from 'node:stream'; +import type { IncomingMessage } from 'node:http'; +import { + createAnalyzeUploadHandler, + requireLocalhostOrigin, +} from '../../src/server/analyze-upload.js'; + +const BOUNDARY = '----gitnexusuploadtest'; + +function multipart( + parts: Array<{ name: string; value?: string; filename?: string; data?: Buffer }>, +): { body: Buffer; headers: Record } { + const chunks: Buffer[] = []; + for (const p of parts) { + chunks.push(Buffer.from(`--${BOUNDARY}\r\n`)); + if (p.filename !== undefined) { + chunks.push( + Buffer.from( + `Content-Disposition: form-data; name="${p.name}"; filename="${p.filename}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n`, + ), + ); + chunks.push(p.data ?? Buffer.alloc(0)); + chunks.push(Buffer.from('\r\n')); + } else { + chunks.push(Buffer.from(`Content-Disposition: form-data; name="${p.name}"\r\n\r\n`)); + chunks.push(Buffer.from(p.value ?? '')); + chunks.push(Buffer.from('\r\n')); + } + } + chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`)); + return { + body: Buffer.concat(chunks), + headers: { 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }, + }; +} + +function mockReq(parts: Parameters[0]): IncomingMessage { + const { body, headers } = multipart(parts); + const r = Readable.from([body]) as unknown as IncomingMessage & { headers: typeof headers }; + r.headers = headers; + return r; +} + +interface MockRes { + statusCode: number; + body: unknown; + status: (c: number) => MockRes; + json: (b: unknown) => MockRes; +} +function mockRes(): MockRes { + const res = { statusCode: 0, body: undefined as unknown } as MockRes; + res.status = (c: number) => { + res.statusCode = c; + return res; + }; + res.json = (b: unknown) => { + res.body = b; + return res; + }; + return res; +} + +// Track promoted upload dirs created by the real ingest+promote so we clean up. +const promoted: string[] = []; +afterEach(async () => { + while (promoted.length) { + await fs.rm(promoted.pop()!, { recursive: true, force: true }).catch(() => {}); + } +}); + +function uniqueTop(): string { + return `uptest_${Math.floor(Math.random() * 1e9).toString(36)}`; +} + +describe('createAnalyzeUploadHandler', () => { + it('ingests, promotes the inner folder, and launches analysis (202)', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => ({ id: 'job-1', status: 'queued' })); + const launch = vi.fn((_j, dir: string) => promoted.push(dir)); + const handler = createAnalyzeUploadHandler({ createJob, launch }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/a.js`, `${top}/sub/b.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('alpha') }, + { name: 'files', filename: 'blob', data: Buffer.from('beta') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(202); + expect((res.body as { jobId: string }).jobId).toBe('job-1'); + expect(createJob).toHaveBeenCalledOnce(); + expect(launch).toHaveBeenCalledOnce(); + const dir = launch.mock.calls[0][1] as string; + const opts = launch.mock.calls[0][2] as { registryName: string }; + // Inner folder promoted: contents live directly under the upload dir. + expect(await fs.readFile(path.join(dir, 'a.js'), 'utf8')).toBe('alpha'); + expect(await fs.readFile(path.join(dir, 'sub', 'b.js'), 'utf8')).toBe('beta'); + expect(opts.registryName).toBe(top); + expect(createJob.mock.calls[0][0].repoPath).toBe(dir); + }); + + it('maps a busy job (createJob throws "already in progress") to 409 and promotes nothing', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => { + throw new Error('Analysis already in progress for another repository'); + }); + const launch = vi.fn((_j, dir: string) => promoted.push(dir)); + const handler = createAnalyzeUploadHandler({ createJob, launch }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/a.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(409); + expect(launch).not.toHaveBeenCalled(); + // Nothing promoted onto disk. + const { UPLOAD_ROOT } = await import('../../src/server/upload-paths.js'); + await expect(fs.access(path.join(UPLOAD_ROOT, top))).rejects.toBeTruthy(); + }); + + it('rejects a traversal path in the manifest (400) without launching', async () => { + const createJob = vi.fn(() => ({ id: 'j', status: 'queued' })); + const launch = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify(['../escape.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(400); + expect(createJob).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + }); + + it('rejects an un-nameable top folder (Windows-reserved → 400)', async () => { + const createJob = vi.fn(() => ({ id: 'j', status: 'queued' })); + const launch = vi.fn(); + const handler = createAnalyzeUploadHandler({ createJob, launch }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify(['CON/a.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(400); + expect(launch).not.toHaveBeenCalled(); + }); + + it('strips a crafted .gitnexus index from the promoted upload', async () => { + const top = uniqueTop(); + const createJob = vi.fn(() => ({ id: 'job-x', status: 'queued' })); + const launch = vi.fn((_j, dir: string) => promoted.push(dir)); + const handler = createAnalyzeUploadHandler({ createJob, launch }); + + const res = mockRes(); + await handler( + mockReq([ + { name: 'manifest', value: JSON.stringify([`${top}/.gitnexus/meta.json`, `${top}/a.js`]) }, + { name: 'files', filename: 'blob', data: Buffer.from('{"evil":true}') }, + { name: 'files', filename: 'blob', data: Buffer.from('real') }, + ]) as never, + res as never, + ); + + expect(res.statusCode).toBe(202); + const dir = launch.mock.calls[0][1] as string; + await expect(fs.access(path.join(dir, '.gitnexus'))).rejects.toBeTruthy(); + expect(await fs.readFile(path.join(dir, 'a.js'), 'utf8')).toBe('real'); + }); +}); + +describe('requireLocalhostOrigin', () => { + function call(origin: string | undefined): { passed: boolean; status: number } { + let passed = false; + let status = 0; + const req = { headers: origin === undefined ? {} : { origin } } as never; + const res = { + status: (c: number) => { + status = c; + return { json: () => {} }; + }, + } as never; + requireLocalhostOrigin(req, res, () => { + passed = true; + }); + return { passed, status }; + } + + it('passes localhost / 127.0.0.1 / no-origin', () => { + expect(call('http://localhost:5173').passed).toBe(true); + expect(call('http://127.0.0.1:4747').passed).toBe(true); + expect(call(undefined).passed).toBe(true); + }); + + it('rejects a public/cross origin with 403', () => { + const r = call('https://gitnexus.vercel.app'); + expect(r.passed).toBe(false); + expect(r.status).toBe(403); + }); +}); diff --git a/gitnexus/test/unit/api-fs-list.test.ts b/gitnexus/test/unit/api-fs-list.test.ts deleted file mode 100644 index 0cb69157c..000000000 --- a/gitnexus/test/unit/api-fs-list.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import path from 'node:path'; -import fs from 'node:fs/promises'; -import os from 'node:os'; -import { handleFsListRequest } from '../../src/server/api.js'; - -let tmpRoot: string; - -beforeAll(async () => { - tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-api-fs-list-test-')); - await fs.mkdir(path.join(tmpRoot, 'alpha')); - await fs.mkdir(path.join(tmpRoot, 'beta')); - await fs.writeFile(path.join(tmpRoot, 'file.txt'), 'hello\n', 'utf-8'); -}); - -afterAll(async () => { - await fs.rm(tmpRoot, { recursive: true, force: true }); -}); - -const invoke = async (query: Record): Promise<{ status: number; body: any }> => { - let capturedStatus = 200; - let capturedBody: any = undefined; - const res = { - status(code: number) { - capturedStatus = code; - return this; - }, - json(body: any) { - capturedBody = body; - }, - }; - await handleFsListRequest({ query }, res); - return { status: capturedStatus, body: capturedBody }; -}; - -describe('GET /api/fs/list — handleFsListRequest', () => { - it('lists only subdirectories, sorted alphabetically', async () => { - const { status, body } = await invoke({ dir: tmpRoot }); - expect(status).toBe(200); - expect(body.entries).toEqual([{ name: 'alpha' }, { name: 'beta' }]); - }); - - it('excludes files from the listing', async () => { - const { body } = await invoke({ dir: tmpRoot }); - const names = body.entries.map((e: { name: string }) => e.name); - expect(names).not.toContain('file.txt'); - }); - - it('returns empty entries for an empty directory', async () => { - const { status, body } = await invoke({ dir: path.join(tmpRoot, 'alpha') }); - expect(status).toBe(200); - expect(body.entries).toEqual([]); - }); - - it('defaults to / when dir is omitted (linux server)', async () => { - const { status } = await invoke({}); - // On Linux root / is the filesystem root; on Windows path.normalize('/') - // normalizes to \ while path.resolve('/') resolves to the CWD drive root. - // Our guard skips the traversal check for bare root paths. - expect(status).toBe(200); - }); - - it('returns 400 for a relative path', async () => { - const { status, body } = await invoke({ dir: 'relative/path' }); - expect(status).toBe(400); - expect(body.error).toMatch(/absolute path/); - }); - - it('returns 400 for a non-canonical absolute path', async () => { - // On Windows, `/foo` is absolute but normalize(\foo) !== resolve(D:\foo) - // On Linux, construct a path with redundant separators that stays absolute - const nonCanonical = process.platform === 'win32' ? '/nonexistent' : `${tmpRoot}/./alpha`; - const { status } = await invoke({ dir: nonCanonical }); - if (process.platform === 'win32') { - expect(status).toBe(400); - } else { - // On Linux, normalize and resolve agree for all absolute paths - expect(status).toBe(200); - } - }); - - it('returns 404 for a non-existent directory', async () => { - const { status, body } = await invoke({ dir: path.join(tmpRoot, 'nonexistent') }); - expect(status).toBe(404); - expect(body.error).toMatch(/not found/i); - }); - - it('returns 400 when dir is an array (type confusion)', async () => { - const { status, body } = await invoke({ dir: [tmpRoot, '/etc'] }); - expect(status).toBe(400); - expect(body.error).toMatch(/single string/); - }); - - it('route source is wired with createRouteLimiter', async () => { - const source = await fs.readFile( - path.join(import.meta.dirname, '..', '..', 'src', 'server', 'api.ts'), - 'utf-8', - ); - expect(source).toMatch(/app\.get\('\/api\/fs\/list',\s*createRouteLimiter\(\)/); - }); -}); diff --git a/gitnexus/test/unit/upload-ingest.test.ts b/gitnexus/test/unit/upload-ingest.test.ts new file mode 100644 index 000000000..7619d744b --- /dev/null +++ b/gitnexus/test/unit/upload-ingest.test.ts @@ -0,0 +1,177 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import { Readable } from 'node:stream'; +import type { IncomingMessage } from 'node:http'; +import { + resolveContainedDest, + ingestUpload, + DEFAULT_INGEST_LIMITS, +} from '../../src/server/upload-ingest.js'; + +// ── resolveContainedDest (pure sanitizer — the load-bearing control) ────────── + +describe('resolveContainedDest', () => { + const ROOT = path.resolve('/tmp/gitnexus-sandbox'); + + it('contains a legitimate nested path under the root', () => { + expect(resolveContainedDest(ROOT, 'myrepo/src/index.js')).toBe( + path.join(ROOT, 'myrepo', 'src', 'index.js'), + ); + }); + + it('allows spaces in names (regression: NUL check must not reject spaces)', () => { + expect(resolveContainedDest(ROOT, 'my repo/a b.js')).toBe(path.join(ROOT, 'my repo', 'a b.js')); + }); + + it.each([ + ['parent traversal', '../../etc/passwd'], + ['mid traversal', 'a/../../b'], + ['dot-dot segment', 'a/../b'], + ['dot segment', 'a/./b'], + ['absolute path', '/etc/shadow'], + ['backslash', 'a\\b'], + ['empty', ''], + ])('rejects %s', (_label, rel) => { + expect(() => resolveContainedDest(ROOT, rel)).toThrow(); + }); + + it('rejects a NUL byte', () => { + expect(() => resolveContainedDest(ROOT, `a${String.fromCharCode(0)}b/x.js`)).toThrow(); + }); + + it('rejects non-string input', () => { + // @ts-expect-error testing runtime guard + expect(() => resolveContainedDest(ROOT, ['a', 'b'])).toThrow(); + }); + + it('rejects an over-deep path (>64 segments)', () => { + const deep = Array.from({ length: 70 }, (_, i) => `d${i}`).join('/') + '/f.js'; + expect(() => resolveContainedDest(ROOT, deep)).toThrow(); + }); + + it('rejects an over-long path (>4096 chars)', () => { + const long = 'a/'.repeat(2100) + 'f.js'; + expect(() => resolveContainedDest(ROOT, long)).toThrow(); + }); + + it('rejects a sibling-prefix escape (root + sep, not bare startsWith)', () => { + // A rel that would resolve to a sibling dir sharing the root's string prefix. + expect(() => resolveContainedDest(ROOT, '../gitnexus-sandbox-evil/x.js')).toThrow(); + }); +}); + +// ── ingestUpload (multipart streaming + containment + caps + cleanup) ────────── + +const BOUNDARY = '----gitnexustestboundary'; + +function multipart( + parts: Array<{ name: string; value?: string; filename?: string; data?: Buffer }>, +): { body: Buffer; headers: Record } { + const chunks: Buffer[] = []; + for (const p of parts) { + chunks.push(Buffer.from(`--${BOUNDARY}\r\n`)); + if (p.filename !== undefined) { + chunks.push( + Buffer.from( + `Content-Disposition: form-data; name="${p.name}"; filename="${p.filename}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n`, + ), + ); + chunks.push(p.data ?? Buffer.alloc(0)); + chunks.push(Buffer.from('\r\n')); + } else { + chunks.push(Buffer.from(`Content-Disposition: form-data; name="${p.name}"\r\n\r\n`)); + chunks.push(Buffer.from(p.value ?? '')); + chunks.push(Buffer.from('\r\n')); + } + } + chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`)); + return { + body: Buffer.concat(chunks), + headers: { 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }, + }; +} + +function mockReq(body: Buffer, headers: Record): IncomingMessage { + const r = Readable.from([body]) as unknown as IncomingMessage & { headers: typeof headers }; + r.headers = headers; + return r; +} + +const staged: string[] = []; +afterEach(async () => { + while (staged.length) { + await fs.rm(staged.pop()!, { recursive: true, force: true }).catch(() => {}); + } +}); + +describe('ingestUpload', () => { + it('writes a manifest-described tree into the sandbox and returns its shape', async () => { + const { body, headers } = multipart([ + { name: 'manifest', value: JSON.stringify(['myrepo/a.js', 'myrepo/sub/b.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('alpha') }, + { name: 'files', filename: 'blob', data: Buffer.from('beta') }, + ]); + const result = await ingestUpload(mockReq(body, headers)); + staged.push(result.stageRoot); + + expect(result.fileCount).toBe(2); + expect(result.topLevelName).toBe('myrepo'); + expect(result.totalBytes).toBe('alpha'.length + 'beta'.length); + expect(await fs.readFile(path.join(result.stageRoot, 'myrepo', 'a.js'), 'utf8')).toBe('alpha'); + expect(await fs.readFile(path.join(result.stageRoot, 'myrepo', 'sub', 'b.js'), 'utf8')).toBe( + 'beta', + ); + }); + + it('rejects a traversal path in the manifest and removes the staging dir', async () => { + const { body, headers } = multipart([ + { name: 'manifest', value: JSON.stringify(['../escape.js']) }, + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + ]); + const before = await countStaging(); + await expect(ingestUpload(mockReq(body, headers))).rejects.toMatchObject({ status: 400 }); + // No new staging dir should survive. + expect(await countStaging()).toBeLessThanOrEqual(before); + }); + + it('rejects a file part that arrives before the manifest', async () => { + const { body, headers } = multipart([ + { name: 'files', filename: 'blob', data: Buffer.from('x') }, + { name: 'manifest', value: JSON.stringify(['a/x.js']) }, + ]); + await expect(ingestUpload(mockReq(body, headers))).rejects.toThrow(/Manifest must precede/); + }); + + it('rejects when total bytes exceed the cap (413)', async () => { + const { body, headers } = multipart([ + { name: 'manifest', value: JSON.stringify(['r/big.bin']) }, + { name: 'files', filename: 'blob', data: Buffer.alloc(64, 1) }, + ]); + await expect(ingestUpload(mockReq(body, headers), { maxTotalBytes: 16 })).rejects.toMatchObject( + { status: 413 }, + ); + }); + + it('rejects an empty upload (0 files)', async () => { + const { body, headers } = multipart([{ name: 'manifest', value: JSON.stringify([]) }]); + await expect(ingestUpload(mockReq(body, headers))).rejects.toThrow(); + }); + + it('exposes sane default caps', () => { + expect(DEFAULT_INGEST_LIMITS.maxTotalBytes).toBe(250 * 1024 * 1024); + expect(DEFAULT_INGEST_LIMITS.maxFiles).toBe(20000); + }); +}); + +async function countStaging(): Promise { + const root = path.resolve(path.join(os.homedir(), '.gitnexus', 'uploads')); + try { + const entries = await fs.readdir(root); + return entries.filter((e) => e.startsWith('.staging-')).length; + } catch { + return 0; + } +} diff --git a/gitnexus/test/unit/upload-sweep.test.ts b/gitnexus/test/unit/upload-sweep.test.ts new file mode 100644 index 000000000..118f75ee0 --- /dev/null +++ b/gitnexus/test/unit/upload-sweep.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import { sweepStaleUploads } from '../../src/server/upload-sweep.js'; + +let root: string; + +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-sweep-test-')); +}); +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }).catch(() => {}); +}); + +describe('sweepStaleUploads', () => { + it('removes stale staging dirs but keeps recent ones and non-staging dirs', async () => { + await fs.mkdir(path.join(root, '.staging-old')); + await fs.mkdir(path.join(root, '.staging-new')); + await fs.mkdir(path.join(root, 'myrepo')); // a promoted (persistent) upload dir + + const now = 1_000_000_000_000; + // Age the "old" staging dir well past the threshold. + const old = new Date(now - 10 * 60 * 60 * 1000); + await fs.utimes(path.join(root, '.staging-old'), old, old); + const recent = new Date(now - 60 * 1000); + await fs.utimes(path.join(root, '.staging-new'), recent, recent); + + const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 }); + + expect(removed).toHaveLength(1); + expect(removed[0]).toContain('.staging-old'); + await expect(fs.access(path.join(root, '.staging-old'))).rejects.toBeTruthy(); + // Recent staging and the promoted repo dir survive. + await expect(fs.access(path.join(root, '.staging-new'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(root, 'myrepo'))).resolves.toBeUndefined(); + }); + + it('tolerates a missing root', async () => { + const { removed } = await sweepStaleUploads({ root: path.join(root, 'does-not-exist') }); + expect(removed).toEqual([]); + }); +});