scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-03 02:21:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix(web): replace server-side directory picker with secure folder upload

Browse source 
PR #1850 review found the new GET /api/fs/list directory-browsing endpoint
enumerated any absolute server path (CodeQL js/path-injection, plus a DoS and
cross-origin enumeration via the CORS/PNA allow-list). Browsers can't hand the
server an absolute path, so rather than harden the endpoint, remove it and
upload the folder instead — webkitdirectory exposes the file contents.

- Add POST /api/analyze/upload: busboy-streamed multipart ingest into an
  mkdtemp sandbox under UPLOAD_ROOT with resolve-then-contain write
  sanitization, hard size/count/dir caps, manifest-first ordering, and
  guaranteed cleanup; promote (atomic same-filesystem rename, no EXDEV) and
  analyze via the shared job/worker machinery, never returning a server path.
- Frontend: <input webkitdirectory> upload flow with client-side filtering
  (.git/node_modules/build), XHR progress, accessibility, en/zh-CN i18n.
- Remove /api/fs/list + handleFsListRequest, DirectoryPicker, listDirectories
  and their tests.
- Harden the adjacent /api/analyze {path} route: localhost-only CORS on write
  routes + realpath/exists/isDir validation replacing the inert
  normalize!==resolve guard.
- Extend DELETE /api/repo cleanup to upload dirs (by entry.path) and add a
  startup sweep for orphaned staging dirs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gergo Magyar 2026-06-09 09:03:08 +00:00
parent e4ed5dfe84
commit 9c46cc0387
21 changed files with 1660 additions and 980 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

325
gitnexus-web/e2e/directory-picker.spec.ts
View file

@ -1,325 +0,0 @@
import { test, expect } from '@playwright/test';
/**
* E2E tests for the server-side directory picker (issue #1518).
*
* All tests mock the backend at the network level so they don't
* require a live gitnexus server. The /api/fs/list endpoint is
* intercepted to return controlled directory structures.
*/
const BACKEND_URL = 'http://localhost:4747';
/** Standard backend mocks needed to reach the analyze form. */
async function mockBackendForAnalyzeForm(page: import('@playwright/test').Page) {
await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] }));
await page.route(`${BACKEND_URL}/api/info`, (route) =>
route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }),
);
await page.route(`${BACKEND_URL}/api/heartbeat`, (route) =>
route.fulfill({
status: 200,
headers: { 'Content-Type': 'text/event-stream' },
body: ':ok\n\n',
}),
);
}
/** Navigate to the Local Folder tab in the analyze form. */
async function openLocalFolderTab(page: import('@playwright/test').Page) {
await page.goto('/');
await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 });
await page.getByRole('tab', { name: 'Local Folder' }).click();
}
// ── Directory picker: open and display ────────────────────────────────────
test.describe('Directory picker — open and display', () => {
test.beforeEach(async ({ page }) => {
await mockBackendForAnalyzeForm(page);
await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => {
const url = new URL(route.request().url());
const dir = url.searchParams.get('dir') ?? '/';
if (dir === '/') {
return route.fulfill({
json: { entries: [{ name: 'workspace' }, { name: 'data' }, { name: 'home' }] },
});
}
if (dir === '/workspace') {
return route.fulfill({
json: { entries: [{ name: 'my-project' }, { name: 'another-repo' }] },
});
}
if (dir === '/workspace/my-project') {
return route.fulfill({ json: { entries: [] } });
}
return route.fulfill({ json: { entries: [] } });
});
});
test('clicking Browse opens the directory picker modal', async ({ page }, testInfo) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
await expect(page.locator('[data-testid="directory-picker-modal"]')).toBeVisible({
timeout: 5_000,
});
await page.screenshot({ path: testInfo.outputPath('picker-open.png') });
});
test('picker shows root directories from /api/fs/list', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
await expect(page.locator('[data-testid="dir-entry-workspace"]')).toBeVisible({
timeout: 5_000,
});
await expect(page.locator('[data-testid="dir-entry-data"]')).toBeVisible();
await expect(page.locator('[data-testid="dir-entry-home"]')).toBeVisible();
});
test('current path shows / at root', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/');
});
});
// ── Directory picker: navigation ──────────────────────────────────────────
test.describe('Directory picker — navigation', () => {
test.beforeEach(async ({ page }) => {
await mockBackendForAnalyzeForm(page);
await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => {
const url = new URL(route.request().url());
const dir = url.searchParams.get('dir') ?? '/';
if (dir === '/') {
return route.fulfill({
json: { entries: [{ name: 'workspace' }, { name: 'data' }] },
});
}
if (dir === '/workspace') {
return route.fulfill({
json: { entries: [{ name: 'my-project' }, { name: 'another-repo' }] },
});
}
if (dir === '/workspace/my-project') {
return route.fulfill({ json: { entries: [{ name: 'src' }] } });
}
return route.fulfill({ json: { entries: [] } });
});
});
test('clicking a directory navigates into it', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
await page.locator('[data-testid="dir-entry-workspace"]').click();
await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({
timeout: 5_000,
});
await expect(page.locator('[data-testid="dir-entry-another-repo"]')).toBeVisible();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace');
});
test('breadcrumb shows path segments after navigation', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
await page.locator('[data-testid="dir-entry-workspace"]').click();
await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({
timeout: 5_000,
});
// Breadcrumb should show "workspace" segment as a clickable button
const modal = page.locator('[data-testid="directory-picker-modal"]');
await expect(modal.getByRole('button', { name: 'workspace', exact: true })).toBeVisible();
});
test('clicking breadcrumb segment navigates back', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
// Navigate: / → /workspace → /workspace/my-project
await page.locator('[data-testid="dir-entry-workspace"]').click();
await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({
timeout: 5_000,
});
await page.locator('[data-testid="dir-entry-my-project"]').click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText(
'/workspace/my-project',
{ timeout: 5_000 },
);
// Click breadcrumb "workspace" to go back to /workspace
const modal = page.locator('[data-testid="directory-picker-modal"]');
const breadcrumbSegments = modal.locator('button').filter({ hasText: 'workspace' });
await breadcrumbSegments.first().click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace', {
timeout: 5_000,
});
await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible();
});
test('home button navigates to root', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
// Navigate into /workspace
await page.locator('[data-testid="dir-entry-workspace"]').click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace', {
timeout: 5_000,
});
// Click home icon
await page.locator('[data-testid="directory-picker-home"]').click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/', {
timeout: 5_000,
});
await expect(page.locator('[data-testid="dir-entry-workspace"]')).toBeVisible();
});
});
// ── Directory picker: selection ───────────────────────────────────────────
test.describe('Directory picker — selection', () => {
test.beforeEach(async ({ page }) => {
await mockBackendForAnalyzeForm(page);
await page.route(`${BACKEND_URL}/api/fs/list*`, (route) => {
const url = new URL(route.request().url());
const dir = url.searchParams.get('dir') ?? '/';
if (dir === '/') {
return route.fulfill({ json: { entries: [{ name: 'workspace' }] } });
}
if (dir === '/workspace') {
return route.fulfill({ json: { entries: [{ name: 'my-project' }] } });
}
return route.fulfill({ json: { entries: [] } });
});
});
test('selecting a folder populates the local path input', async ({ page }, testInfo) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
// Navigate to /workspace
await page.locator('[data-testid="dir-entry-workspace"]').click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText('/workspace', {
timeout: 5_000,
});
// Click "Select this folder"
await page.locator('[data-testid="directory-picker-select"]').click();
// Modal should close
await expect(page.locator('[data-testid="directory-picker-modal"]')).not.toBeVisible();
// The local path input should contain the selected path
const pathInput = page.locator('input[type="text"]');
await expect(pathInput).toHaveValue('/workspace');
await page.screenshot({ path: testInfo.outputPath('path-populated.png') });
});
test('selected path is an absolute path', async ({ page }) => {
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
await page.locator('[data-testid="dir-entry-workspace"]').click();
await expect(page.locator('[data-testid="dir-entry-my-project"]')).toBeVisible({
timeout: 5_000,
});
await page.locator('[data-testid="dir-entry-my-project"]').click();
await expect(page.locator('[data-testid="directory-picker-path"]')).toHaveText(
'/workspace/my-project',
{ timeout: 5_000 },
);
await page.locator('[data-testid="directory-picker-select"]').click();
const pathInput = page.locator('input[type="text"]');
await expect(pathInput).toHaveValue('/workspace/my-project');
});
test('closing the modal does not change the path input', async ({ page }) => {
await openLocalFolderTab(page);
// Type a path manually first
const pathInput = page.locator('input[type="text"]');
await pathInput.fill('/my/custom/path');
// Open and close the picker without selecting
await page.locator('[data-testid="browse-server-dirs"]').click();
await expect(page.locator('[data-testid="directory-picker-modal"]')).toBeVisible({
timeout: 5_000,
});
// Click the backdrop to close
await page
.locator('[data-testid="directory-picker-modal"]')
.locator('..')
.locator('div')
.first()
.click({ position: { x: 5, y: 5 }, force: true });
// Path input should keep the original value
await expect(pathInput).toHaveValue('/my/custom/path');
});
});
// ── Directory picker: edge cases ──────────────────────────────────────────
test.describe('Directory picker — edge cases', () => {
test('shows empty state for a directory with no subdirectories', async ({ page }) => {
await mockBackendForAnalyzeForm(page);
await page.route(`${BACKEND_URL}/api/fs/list*`, (route) =>
route.fulfill({ json: { entries: [] } }),
);
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
const modal = page.locator('[data-testid="directory-picker-modal"]');
await expect(modal.getByText('This directory is empty.')).toBeVisible({ timeout: 5_000 });
});
test('shows error when /api/fs/list fails', async ({ page }) => {
await mockBackendForAnalyzeForm(page);
await page.route(`${BACKEND_URL}/api/fs/list*`, (route) =>
route.fulfill({ status: 500, json: { error: 'Internal server error' } }),
);
await openLocalFolderTab(page);
await page.locator('[data-testid="browse-server-dirs"]').click();
const modal = page.locator('[data-testid="directory-picker-modal"]');
await expect(modal.locator('text=Go back')).toBeVisible({ timeout: 5_000 });
});
test('manual path typing still works without opening picker', async ({ page }) => {
await mockBackendForAnalyzeForm(page);
await openLocalFolderTab(page);
// Type a path manually
const pathInput = page.locator('input[type="text"]');
await pathInput.fill('/workspace/my-repo');
// The input should have the typed path
await expect(pathInput).toHaveValue('/workspace/my-repo');
// The Analyze button should be enabled
const analyzeBtn = page.getByRole('button', { name: /Analyze Repository/ });
await expect(analyzeBtn).toBeEnabled();
});
});

66
gitnexus-web/e2e/folder-upload.spec.ts Normal file
View file

@ -0,0 +1,66 @@
import { test, expect } from '@playwright/test';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
/**
* E2E for the browser folder-upload flow (replaces the removed server-side
* directory picker). Mocks the backend so no live gitnexus server is needed.
*/
const BACKEND_URL = 'http://localhost:4747';
let fixtureDir: string;
test.beforeAll(() => {
// A tiny "repo" folder; Playwright sets webkitRelativePath = <folder>/<file>.
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-upload-e2e-'));
fixtureDir = path.join(root, 'myrepo');
fs.mkdirSync(path.join(fixtureDir, 'src'), { recursive: true });
fs.writeFileSync(path.join(fixtureDir, 'README.md'), '# hi\n');
fs.writeFileSync(path.join(fixtureDir, 'src', 'index.ts'), 'export const x = 1;\n');
});
test.beforeEach(async ({ page }) => {
await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] }));
await page.route(`${BACKEND_URL}/api/info`, (route) =>
route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }),
);
await page.route(`${BACKEND_URL}/api/heartbeat`, (route) =>
route.fulfill({
status: 200,
headers: { 'Content-Type': 'text/event-stream' },
body: ':ok\n\n',
}),
);
});
test('uploading a folder posts a multipart upload and starts analysis', async ({ page }) => {
let uploadContentType = '';
await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => {
uploadContentType = route.request().headers()['content-type'] ?? '';
await route.fulfill({ json: { jobId: 'job-e2e', status: 'analyzing' } });
});
// SSE progress → immediately complete.
await page.route(`${BACKEND_URL}/api/analyze/job-e2e/progress`, (route) =>
route.fulfill({
status: 200,
headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' },
body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n',
}),
);
await page.goto('/');
await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 });
await page.getByRole('tab', { name: 'Local Folder' }).click();
await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible();
// Select the fixture folder via the hidden webkitdirectory input.
await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir);
// The upload endpoint should be hit with a multipart body, and the UI should
// leave the input phase (upload button no longer shown).
await expect.poll(() => uploadContentType).toContain('multipart/form-data');
await expect(page.locator('[data-testid="upload-folder"]')).toBeHidden({ timeout: 10_000 });
});

4
gitnexus-web/e2e/onboarding.spec.ts
View file

@ -218,8 +218,8 @@ test.describe('Flow 3: Analyze form', () => {
// Switch to Local Folder tab
await page.getByRole('tab', { name: 'Local Folder' }).click();
// Browse button should be visible (server-side directory picker)
await expect(page.locator('[data-testid="browse-server-dirs"]')).toBeVisible();
// Upload-a-folder button should be visible (browser folder upload)
await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible();
await page.screenshot({ path: testInfo.outputPath('local-folder-tab.png') });
});

217
gitnexus-web/src/components/DirectoryPicker.tsx
View file

@ -1,217 +0,0 @@
import { useState, useEffect, useCallback, useRef } from 'react';
import { Folder, ChevronRight, Home, X, Loader2, AlertCircle, Check } from '@/lib/lucide-icons';
import { listDirectories, type DirEntry } from '../services/backend-client';
import { useTranslation } from 'react-i18next';
interface DirectoryPickerProps {
open: boolean;
onClose: () => void;
onSelect: (absolutePath: string) => void;
initialDir?: string;
}
export const DirectoryPicker = ({ open, onClose, onSelect, initialDir }: DirectoryPickerProps) => {
const { t } = useTranslation('onboarding');
const [currentDir, setCurrentDir] = useState(initialDir ?? '/');
const [entries, setEntries] = useState<DirEntry[]>([]);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [isVisible, setIsVisible] = useState(false);
const panelRef = useRef<HTMLDivElement>(null);
const fetchEntries = useCallback(async (dir: string) => {
setLoading(true);
setError(null);
try {
const result = await listDirectories(dir);
setEntries(result.entries);
setCurrentDir(dir);
} catch (err: unknown) {
setError(err instanceof Error ? err.message : 'Failed to list directory');
} finally {
setLoading(false);
}
}, []);
useEffect(() => {
if (open) {
fetchEntries(initialDir ?? '/');
requestAnimationFrame(() => setIsVisible(true));
} else {
setIsVisible(false);
}
}, [open, initialDir, fetchEntries]);
useEffect(() => {
if (!open) return;
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'Escape') {
e.stopPropagation();
onClose();
}
};
document.addEventListener('keydown', handleKeyDown);
return () => document.removeEventListener('keydown', handleKeyDown);
}, [open, onClose]);
useEffect(() => {
if (open && panelRef.current) {
panelRef.current.focus();
}
}, [open]);
if (!open) return null;
const segments = currentDir.split('/').filter(Boolean);
const navigateTo = (dir: string) => {
fetchEntries(dir);
};
const handleBreadcrumbClick = (index: number) => {
const target = '/' + segments.slice(0, index + 1).join('/');
navigateTo(target);
};
return (
<div className="fixed inset-0 z-50 flex items-center justify-center">
<div
className={`absolute inset-0 bg-black/60 backdrop-blur-sm transition-opacity duration-200 ${isVisible ? 'opacity-100' : 'opacity-0'}`}
onClick={onClose}
aria-hidden="true"
/>
<div
ref={panelRef}
data-testid="directory-picker-modal"
role="dialog"
aria-modal="true"
aria-label={t('repoAnalyzer.directoryPicker.title')}
tabIndex={-1}
className={`relative mx-4 flex w-full max-w-lg flex-col overflow-hidden rounded-2xl border border-border-subtle bg-surface shadow-2xl transition-all duration-200 outline-none ${isVisible ? 'scale-100 opacity-100' : 'scale-95 opacity-0'}`}
style={{ maxHeight: '70vh' }}
>
{/* Header */}
<div className="flex items-center justify-between border-b border-border-subtle px-5 py-4">
<h3 className="text-sm font-semibold text-text-primary">
{t('repoAnalyzer.directoryPicker.title')}
</h3>
<button
onClick={onClose}
aria-label="Close"
className="rounded-md p-1.5 text-text-muted transition-colors hover:text-text-primary focus-visible:ring-2 focus-visible:ring-accent/40 focus-visible:outline-none"
>
<X className="h-4 w-4" />
</button>
</div>
{/* Breadcrumb */}
<nav
aria-label="Directory breadcrumb"
className="flex items-center gap-1 overflow-x-auto border-b border-border-subtle bg-elevated/50 px-5 py-2"
>
<button
data-testid="directory-picker-home"
onClick={() => navigateTo('/')}
aria-label="Root directory"
className="shrink-0 rounded-md p-1.5 text-text-muted transition-colors hover:text-accent focus-visible:ring-2 focus-visible:ring-accent/40 focus-visible:outline-none"
>
<Home className="h-3.5 w-3.5" />
</button>
{segments.map((seg, i) => (
<span key={i} className="flex shrink-0 items-center gap-1">
<ChevronRight className="h-3 w-3 text-text-muted/50" aria-hidden="true" />
<button
onClick={() => handleBreadcrumbClick(i)}
aria-current={i === segments.length - 1 ? 'location' : undefined}
className={`rounded-md px-1.5 py-1 font-mono text-xs transition-colors focus-visible:ring-2 focus-visible:ring-accent/40 focus-visible:outline-none ${
i === segments.length - 1
? 'font-medium text-text-primary'
: 'text-text-muted hover:text-accent'
}`}
>
{seg}
</button>
</span>
))}
</nav>
{/* Directory listing */}
<div
data-testid="directory-listing"
className="min-h-[200px] flex-1 overflow-y-auto px-2 py-2"
>
{loading && (
<div className="flex items-center justify-center py-12" role="status">
<Loader2 className="h-5 w-5 animate-spin text-text-muted" />
<span className="sr-only">Loading directories...</span>
</div>
)}
{error && !loading && (
<div className="flex flex-col items-center gap-2 py-12 text-center" role="alert">
<AlertCircle className="h-5 w-5 text-red-400" />
<p className="text-xs text-red-400">{error}</p>
<button
onClick={() =>
navigateTo(currentDir === '/' ? '/' : currentDir.replace(/\/[^/]+$/, '') || '/')
}
className="mt-1 rounded-md px-2 py-1 text-xs text-text-muted underline transition-colors hover:text-text-secondary focus-visible:ring-2 focus-visible:ring-accent/40 focus-visible:outline-none"
>
{t('repoAnalyzer.directoryPicker.goBack')}
</button>
</div>
)}
{!loading && !error && entries.length === 0 && (
<div className="flex flex-col items-center gap-1 py-12 text-center">
<Folder className="h-5 w-5 text-text-muted/50" />
<p className="text-xs text-text-muted">{t('repoAnalyzer.directoryPicker.empty')}</p>
</div>
)}
{!loading &&
!error &&
entries.map((entry) => {
const target = currentDir === '/' ? `/${entry.name}` : `${currentDir}/${entry.name}`;
return (
<button
key={entry.name}
data-testid={`dir-entry-${entry.name}`}
onClick={() => navigateTo(target)}
className="flex w-full items-center gap-2.5 rounded-lg px-3 py-2.5 text-left transition-colors hover:bg-hover focus-visible:ring-2 focus-visible:ring-accent/40 focus-visible:outline-none active:bg-hover/70"
>
<Folder className="h-4 w-4 shrink-0 text-accent/70" />
<span className="truncate font-mono text-xs text-text-secondary">
{entry.name}
</span>
<ChevronRight
className="ml-auto h-3 w-3 shrink-0 text-text-muted/40"
aria-hidden="true"
/>
</button>
);
})}
</div>
{/* Footer — current path + select button */}
<div className="flex items-center gap-3 border-t border-border-subtle bg-elevated/30 px-5 py-3">
<code
data-testid="directory-picker-path"
className="min-w-0 flex-1 truncate rounded bg-void px-2.5 py-1.5 font-mono text-xs text-text-secondary"
>
{currentDir}
</code>
<button
data-testid="directory-picker-select"
onClick={() => onSelect(currentDir)}
className="flex shrink-0 items-center gap-1.5 rounded-lg bg-accent px-3.5 py-2 text-xs font-medium text-white transition-colors hover:bg-accent/90 focus-visible:ring-2 focus-visible:ring-accent/40 focus-visible:ring-offset-2 focus-visible:ring-offset-surface focus-visible:outline-none active:bg-accent/80"
>
<Check className="h-3.5 w-3.5" />
{t('repoAnalyzer.directoryPicker.select')}
</button>
</div>
</div>
</div>
);
};

141
gitnexus-web/src/components/RepoAnalyzer.tsx
View file

@ -21,10 +21,11 @@ import {
startAnalyze,
cancelAnalyze,
streamAnalyzeProgress,
uploadFolder,
type JobProgress,
} from '../services/backend-client';
import { AnalyzeProgress } from './AnalyzeProgress';
import { DirectoryPicker } from './DirectoryPicker';
import { filterRepoFiles } from '@/lib/upload-filter';
import { useTranslation } from 'react-i18next';
// ── Helpers ──────────────────────────────────────────────────────────────────
@ -167,7 +168,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp
const { t } = useTranslation(['common', 'errors', 'onboarding']);
const inputId = useId();
const [mode, setMode] = useState<InputMode>('github');
const [pickerOpen, setPickerOpen] = useState(false);
const [uploadPercent, setUploadPercent] = useState<number | null>(null);
const [uploadSummary, setUploadSummary] = useState<{ count: number; dropped: number } | null>(
null,
);
const [githubUrl, setGithubUrl] = useState('');
const [gitlabUrl, setGitlabUrl] = useState('');
const [localPath, setLocalPath] = useState('');
@ -183,6 +187,7 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp
const jobIdRef = useRef<string | null>(null);
const sseControllerRef = useRef<AbortController | null>(null);
const completeTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const folderInputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
return () => {
@ -237,8 +242,6 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp
? { url: gitlabUrl.trim() }
: { path: localPath.trim() };
const { jobId } = await startAnalyze(request);
jobIdRef.current = jobId;
setPhase('analyzing');
const nameSource =
mode === 'github'
@ -246,34 +249,67 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp
: mode === 'gitlab'
? gitlabUrl.trim()
: localPath.trim();
const controller = streamAnalyzeProgress(
jobId,
(p) => setProgress(p),
(data) => {
const name =
data.repoName ??
nameSource.split(/[/\\]/).filter(Boolean).at(-1) ??
t('onboarding:repoAnalyzer.defaultRepoName');
setCompletedRepoName(name);
setPhase('done');
sseControllerRef.current = null;
completeTimerRef.current = setTimeout(() => {
completeTimerRef.current = null;
onComplete(name);
}, 1200);
},
(errMsg) => {
setValidationError(errMsg || t('errors:analysisFailed'));
setPhase('error');
},
);
sseControllerRef.current = controller;
trackJob(jobId, nameSource);
} catch (err) {
setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed'));
setPhase('error');
}
};
// Drive an already-created analysis job through the SSE progress stream to
// completion. Shared by the path/URL analyze flow and the folder-upload flow.
const trackJob = (jobId: string, fallbackNameSource: string | null) => {
jobIdRef.current = jobId;
setPhase('analyzing');
const controller = streamAnalyzeProgress(
jobId,
(p) => setProgress(p),
(data) => {
const name =
data.repoName ??
(fallbackNameSource
? fallbackNameSource.split(/[/\\]/).filter(Boolean).at(-1)
: undefined) ??
t('onboarding:repoAnalyzer.defaultRepoName');
setCompletedRepoName(name);
setPhase('done');
sseControllerRef.current = null;
completeTimerRef.current = setTimeout(() => {
completeTimerRef.current = null;
onComplete(name);
}, 1200);
},
(errMsg) => {
setValidationError(errMsg || t('errors:analysisFailed'));
setPhase('error');
},
);
sseControllerRef.current = controller;
};
// Upload a browser-selected folder (webkitdirectory) and start analysis. The
// upload endpoint returns a jobId, which then joins the normal SSE flow.
const handleFolderUpload = async (fileList: FileList) => {
const { files, manifest, droppedCount } = filterRepoFiles(fileList);
if (files.length === 0) {
setValidationError(t('onboarding:repoAnalyzer.upload.empty'));
return;
}
setValidationError(null);
setUploadSummary({ count: files.length, dropped: droppedCount });
setUploadPercent(0);
setPhase('starting');
try {
const { jobId } = await uploadFolder(files, manifest, (pct) => setUploadPercent(pct));
setUploadPercent(null);
trackJob(jobId, null);
} catch (err) {
setUploadPercent(null);
setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed'));
setPhase('error');
}
};
const handleCancel = async () => {
sseControllerRef.current?.abort();
sseControllerRef.current = null;
@ -444,25 +480,54 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp
<Check className="h-3.5 w-3.5 shrink-0 text-emerald-400" />
)}
</div>
{/* Upload a folder from your computer — no server path or mount needed.
The browser can't expose an absolute path, so we upload the files. */}
<input
ref={folderInputRef}
type="file"
// @ts-expect-error -- webkitdirectory is non-standard but widely supported
webkitdirectory=""
multiple
className="hidden"
data-testid="folder-upload-input"
onChange={(e) => {
if (e.target.files && e.target.files.length > 0) {
handleFolderUpload(e.target.files);
}
e.target.value = '';
}}
/>
<button
type="button"
data-testid="browse-server-dirs"
onClick={() => setPickerOpen(true)}
data-testid="upload-folder"
onClick={() => folderInputRef.current?.click()}
disabled={isLoading}
className="flex w-full cursor-pointer items-center justify-center gap-2 rounded-lg border border-border-subtle bg-elevated px-3 py-2 text-xs font-medium text-text-secondary transition-all duration-150 hover:bg-hover hover:text-text-primary disabled:opacity-50"
>
<FolderOpen className="h-3.5 w-3.5" />
{t('onboarding:repoAnalyzer.browseForFolder')}
{t('onboarding:repoAnalyzer.upload.button')}
</button>
<DirectoryPicker
open={pickerOpen}
onClose={() => setPickerOpen(false)}
onSelect={(selectedPath) => {
setLocalPath(selectedPath);
setPickerOpen(false);
setValidationError(null);
}}
/>
{uploadPercent !== null && (
<div role="status" data-testid="upload-progress" className="space-y-1">
<div className="h-1.5 w-full overflow-hidden rounded-full bg-elevated">
<div
className="h-full bg-accent transition-all duration-150"
style={{ width: `${uploadPercent}%` }}
/>
</div>
<p className="text-xs text-text-muted">
{t('onboarding:repoAnalyzer.upload.uploading', { percent: uploadPercent })}
</p>
</div>
)}
{uploadSummary && uploadPercent === null && phase !== 'error' && (
<p className="text-xs text-text-muted" data-testid="upload-summary">
{t('onboarding:repoAnalyzer.upload.selected', {
count: uploadSummary.count,
dropped: uploadSummary.dropped,
})}
</p>
)}
</div>
)}

45
gitnexus-web/src/lib/upload-filter.test.ts Normal file
View file

@ -0,0 +1,45 @@
import { describe, expect, it } from 'vitest';
import { filterRepoFiles, MAX_FILE_BYTES } from './upload-filter';
type FileLike = { name: string; size: number; webkitRelativePath?: string };
function f(webkitRelativePath: string, size = 10): FileLike {
const name = webkitRelativePath.split('/').pop() ?? webkitRelativePath;
return { name, size, webkitRelativePath };
}
describe('filterRepoFiles', () => {
it('keeps source files and builds an order-aligned manifest', () => {
const input = [f('repo/src/index.ts', 100), f('repo/README.md', 50)];
const r = filterRepoFiles(input);
expect(r.files).toHaveLength(2);
expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/README.md']);
expect(r.totalBytes).toBe(150);
expect(r.droppedCount).toBe(0);
});
it('excludes .git / node_modules / build dirs anywhere in the path', () => {
const input = [
f('repo/.git/HEAD'),
f('repo/node_modules/x/index.js'),
f('repo/dist/bundle.js'),
f('repo/src/app.ts'),
f('repo/.gitnexus/meta.json'),
];
const r = filterRepoFiles(input);
expect(r.manifest).toEqual(['repo/src/app.ts']);
expect(r.droppedCount).toBe(4);
});
it('drops files over the per-file size cap', () => {
const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)];
const r = filterRepoFiles(input);
expect(r.manifest).toEqual(['repo/small.ts']);
expect(r.droppedCount).toBe(1);
});
it('falls back to name when webkitRelativePath is absent', () => {
const r = filterRepoFiles([{ name: 'lone.ts', size: 5 }]);
expect(r.manifest).toEqual(['lone.ts']);
});
});

73
gitnexus-web/src/lib/upload-filter.ts Normal file
View file

@ -0,0 +1,73 @@
/**
* Client-side pre-filter for a webkitdirectory folder upload.
*
* Drops VCS metadata, dependency/build directories, and oversized files before
* upload — `.git` alone is often larger than the working tree — so payloads
* stay small and the upload matches what the analyzer actually needs. Produces
* an order-aligned `manifest` of webkitRelativePaths (the server keys on this,
* not the multipart filename, which browsers rewrite).
*/
/** Directory names excluded anywhere in a file's path. */
export const EXCLUDED_DIRS = new Set([
'.git',
'.hg',
'.svn',
'node_modules',
'vendor',
'.venv',
'__pycache__',
'target',
'dist',
'build',
'out',
'.next',
'.nuxt',
'.cache',
'coverage',
'.idea',
'.gitnexus',
]);
/** Per-file size cap; matches the server's per-file limit. */
export const MAX_FILE_BYTES = 25 * 1024 * 1024;
export interface FilterResult {
files: File[];
manifest: string[];
droppedCount: number;
totalBytes: number;
}
type FileLike = Pick<File, 'name' | 'size'> & { webkitRelativePath?: string };
/**
* Filter a webkitdirectory `FileList` (or array) into the files to upload plus
* their relative-path manifest.
*/
export function filterRepoFiles(input: ArrayLike<FileLike>): FilterResult {
const files: File[] = [];
const manifest: string[] = [];
let droppedCount = 0;
let totalBytes = 0;
for (let i = 0; i < input.length; i++) {
const f = input[i];
const rel =
f.webkitRelativePath && f.webkitRelativePath.length > 0 ? f.webkitRelativePath : f.name;
const segments = rel.split('/');
if (segments.some((s) => EXCLUDED_DIRS.has(s))) {
droppedCount++;
continue;
}
if (f.size > MAX_FILE_BYTES) {
droppedCount++;
continue;
}
files.push(f as File);
manifest.push(rel);
totalBytes += f.size;
}
return { files, manifest, droppedCount, totalBytes };
}

11
gitnexus-web/src/locales/en/onboarding.json
View file

@ -61,13 +61,12 @@
"gitlabRepositoryUrl": "GitLab Repository URL",
"gitlabSupported": "Supports GitLab.com and self-hosted GitLab instances.",
"localFolderPath": "Local Folder Path",
"browseForFolder": "Browse server directories",
"hideBackground": "Hide (analysis continues in background)",
"directoryPicker": {
"title": "Browse Server Directories",
"select": "Select this folder",
"empty": "This directory is empty.",
"goBack": "Go back"
"upload": {
"button": "Upload a folder",
"uploading": "Uploading… {{percent}}%",
"selected": "{{count}} files ready ({{dropped}} skipped: .git, node_modules, build output)",
"empty": "No analyzable files found in that folder."
}
}
}

11
gitnexus-web/src/locales/zh-CN/onboarding.json
View file

@ -61,13 +61,12 @@
"gitlabRepositoryUrl": "GitLab 仓库 URL",
"gitlabSupported": "支持 GitLab.com 和自托管 GitLab 实例。",
"localFolderPath": "本地文件夹路径",
"browseForFolder": "浏览服务器目录",
"hideBackground": "隐藏（分析继续在后台进行）",
"directoryPicker": {
"title": "浏览服务器目录",
"select": "选择此文件夹",
"empty": "此目录为空。",
"goBack": "返回上级"
"upload": {
"button": "上传文件夹",
"uploading": "上传中… {{percent}}%",
"selected": "已准备 {{count}} 个文件（已跳过 {{dropped}} 个：.git、node_modules、构建产物）",
"empty": "该文件夹中未找到可分析的文件。"
}
}
}

61
gitnexus-web/src/services/backend-client.ts
View file

@ -755,21 +755,56 @@ export const fetchClusterDetail = async (repo: string, name: string): Promise<un
return response.json();
};
// ── Filesystem API ───────────────────────────────────────────────────────
// ── Upload API ─────────────────────────────────────────────────────────────
export interface DirEntry {
name: string;
}
/**
* Upload a folder (selected via `<input webkitdirectory>`) and start analysis.
* Sends the file blobs plus a JSON `manifest` of their relative paths — the
* multipart filename can't carry the path (browsers strip separators), so the
* manifest is the source of truth. Uses XHR for upload progress. Returns the
* analysis jobId, which the caller drives through the normal SSE flow.
*/
export const uploadFolder = (
files: File[],
manifest: string[],
onProgress?: (percent: number) => void,
): Promise<{ jobId: string; status: string }> => {
return new Promise((resolve, reject) => {
const form = new FormData();
// Manifest MUST precede the file parts (the server enforces this).
form.append('manifest', JSON.stringify(manifest));
for (const f of files) form.append('files', f);
/** List subdirectories at the given absolute server-side path. */
export const listDirectories = async (dir: string): Promise<{ entries: DirEntry[] }> => {
const response = await fetchWithTimeout(
`${_backendUrl}/api/fs/list?dir=${encodeURIComponent(dir)}`,
undefined,
5_000,
);
await assertOk(response);
return response.json() as Promise<{ entries: DirEntry[] }>;
const xhr = new XMLHttpRequest();
xhr.open('POST', `${_backendUrl}/api/analyze/upload`);
xhr.timeout = 5 * 60_000; // up to 5 min for large repos
xhr.upload.onprogress = (e) => {
if (onProgress && e.lengthComputable) {
onProgress(Math.round((e.loaded / e.total) * 100));
}
};
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
try {
resolve(JSON.parse(xhr.responseText) as { jobId: string; status: string });
} catch {
reject(new Error('Invalid server response'));
}
return;
}
let msg = `Upload failed (${xhr.status})`;
try {
const body = JSON.parse(xhr.responseText);
if (body?.error) msg = body.error;
} catch {
/* keep default message */
}
reject(new Error(msg));
};
xhr.onerror = () => reject(new Error('Upload failed: network error'));
xhr.ontimeout = () => reject(new Error('Upload timed out'));
xhr.send(form);
});
};
// ── Analyze API ────────────────────────────────────────────────────────────

31
gitnexus/package-lock.json generated
View file

@ -14,6 +14,7 @@
"@ladybugdb/core": "^0.17.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"@scarf/scarf": "^1.4.0",
"busboy": "^1.6.0",
"cli-progress": "^3.12.0",
"commander": "^14.0.3",
"cors": "^2.8.5",
@ -50,6 +51,7 @@
"gitnexus": "dist/cli/index.js"
},
"devDependencies": {
"@types/busboy": "^1.5.4",
"@types/cli-progress": "^3.11.6",
"@types/cors": "^2.8.17",
"@types/express": "^5.0.6",
@ -1707,6 +1709,16 @@
"@types/node": "*"
}
},
"node_modules/@types/busboy": {
"version": "1.5.4",
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz",
"integrity": "sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/chai": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
@ -2217,6 +2229,17 @@
"node": "18 || 20 || >=22"
}
},
"node_modules/busboy": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
"dependencies": {
"streamsearch": "^1.1.0"
},
"engines": {
"node": ">=10.16.0"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@ -4819,6 +4842,14 @@
"dev": true,
"license": "MIT"
},
"node_modules/streamsearch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",

2
gitnexus/package.json
View file

@ -58,6 +58,7 @@
"@ladybugdb/core": "^0.17.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"@scarf/scarf": "^1.4.0",
"busboy": "^1.6.0",
"cli-progress": "^3.12.0",
"commander": "^14.0.3",
"cors": "^2.8.5",
@ -96,6 +97,7 @@
"tree-sitter-kotlin": "^0.3.8"
},
"devDependencies": {
"@types/busboy": "^1.5.4",
"@types/cli-progress": "^3.11.6",
"@types/cors": "^2.8.17",
"@types/express": "^5.0.6",

156
gitnexus/src/server/analyze-upload.ts Normal file
View file

@ -0,0 +1,156 @@
/**
* POST /api/analyze/upload — analyze a browser folder upload.
*
* Securely ingests the multipart upload into a sandbox (upload-ingest.ts),
* promotes it to a persistent app-controlled directory, and analyzes it via
* the same job/worker machinery as a git clone — never returning a server
* path to the client. Factored as a dependency-injected handler so the job
* machinery (createJob + the worker launcher) can be mocked in unit tests.
*/
import path from 'path';
import fsp from 'fs/promises';
import type { Request, Response } from 'express';
import type { IncomingMessage } from 'http';
import { ingestUpload } from './upload-ingest.js';
import { UPLOAD_ROOT, getUploadDir, deriveUploadName } from './upload-paths.js';
import { BadRequestError } from './validation.js';
export interface UploadJobRef {
id: string;
status: string;
}
export interface AnalyzeUploadDeps {
/** Create (or throw on busy) an analysis job for the given upload dir. */
createJob: (params: { repoPath: string }) => UploadJobRef;
/** Launch the analyze worker against an already-resolved repo directory. */
launch: (job: UploadJobRef, targetPath: string, opts: { registryName: string }) => void;
/** Injectable for tests (defaults to the real ingestUpload). */
ingest?: typeof ingestUpload;
}
/**
* Find an available upload directory name, appending `-2`, `-3`, … on
* collision with an existing upload. Bounded to avoid an unbounded scan.
*/
async function pickAvailableName(base: string): Promise<string> {
for (let i = 0; i < 100; i++) {
const name = i === 0 ? base : `${base}-${i + 1}`;
let dir: string;
try {
dir = getUploadDir(name);
} catch {
continue;
}
try {
await fsp.access(dir);
// exists → try the next suffix
} catch {
return name; // ENOENT → available
}
}
throw new BadRequestError('Could not allocate an upload directory', 409);
}
export function createAnalyzeUploadHandler(deps: AnalyzeUploadDeps) {
const ingest = deps.ingest ?? ingestUpload;
return async function handleAnalyzeUploadRequest(req: Request, res: Response): Promise<void> {
let stageRoot: string | undefined;
let promotedDir: string | undefined;
let launched = false;
try {
const result = await ingest(req as unknown as IncomingMessage);
stageRoot = result.stageRoot;
const baseName = deriveUploadName(result.topLevelName);
if (!baseName) {
throw new BadRequestError('Uploaded folder has no usable name');
}
const finalName = await pickAvailableName(baseName);
const finalDir = getUploadDir(finalName);
// createJob BEFORE promote: a busy server throws → 409 and nothing has
// been moved into place yet, so the staging dir is the only thing to clean.
let job: UploadJobRef;
try {
job = deps.createJob({ repoPath: finalDir });
} catch (err) {
const msg = err instanceof Error ? err.message : '';
if (msg.includes('already in progress')) {
throw new BadRequestError(msg, 409);
}
throw err;
}
// webkitRelativePath prefixes every entry with the picked folder, so the
// real repo root is stageRoot/<topLevelName>. Promote that inner dir.
const innerRoot = path.join(result.stageRoot, result.topLevelName);
try {
if (!(await fsp.stat(innerRoot)).isDirectory()) {
throw new BadRequestError('Upload must be a folder');
}
} catch (err) {
if (err instanceof BadRequestError) throw err;
throw new BadRequestError('Upload must be a folder');
}
// Promote staging → persistent upload dir. Both live under UPLOAD_ROOT's
// filesystem, so this rename stays atomic (no EXDEV).
await fsp.mkdir(UPLOAD_ROOT, { recursive: true });
await fsp.rename(innerRoot, finalDir);
promotedDir = finalDir;
const oldStage = stageRoot;
stageRoot = undefined;
await fsp.rm(oldStage, { recursive: true, force: true }).catch(() => {});
// Drop any crafted index the upload may have carried (a `.gitnexus`
// segment passes containment); the worker will build a fresh one.
await fsp
.rm(path.join(finalDir, '.gitnexus'), { recursive: true, force: true })
.catch(() => {});
deps.launch(job, finalDir, { registryName: finalName });
launched = true;
res.status(202).json({ jobId: job.id, status: job.status });
} catch (err) {
if (stageRoot) {
await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => {});
}
if (promotedDir && !launched) {
await fsp.rm(promotedDir, { recursive: true, force: true }).catch(() => {});
}
if (err instanceof BadRequestError) {
res.status(err.status).json({ error: err.message });
return;
}
res.status(500).json({ error: 'Upload failed' });
}
};
}
/**
* Per-route guard that restricts an endpoint to localhost browser origins.
* Non-browser requests (no Origin header, e.g. curl) pass through. This closes
* cross-origin reach (e.g. the allow-listed public deploy + Private Network
* Access) to write routes without affecting read routes.
*/
export function requireLocalhostOrigin(req: Request, res: Response, next: () => void): void {
const origin = req.headers.origin;
if (origin === undefined) {
next();
return;
}
try {
const hostname = new URL(origin).hostname;
if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') {
next();
return;
}
} catch {
/* malformed origin → reject */
}
res.status(403).json({ error: 'This endpoint is restricted to localhost origins' });
}

537
gitnexus/src/server/api.ts
View file

@ -35,6 +35,9 @@ import { fileURLToPath, pathToFileURL } from 'url';
import { JobManager } from './analyze-job.js';
import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js';
import { extractRepoName, getCloneDir, cloneOrPull } from './git-clone.js';
import { createAnalyzeUploadHandler, requireLocalhostOrigin } from './analyze-upload.js';
import { UPLOAD_ROOT } from './upload-paths.js';
import { sweepStaleUploads } from './upload-sweep.js';
import { logger, flushLoggerSync } from '../core/logger.js';
const _require = createRequire(import.meta.url);
@ -563,66 +566,6 @@ const requestedRepo = (req: express.Request): string | undefined => {
return undefined;
};
/**
* Handle a GET /api/fs/list request. Lists subdirectories at a given
* absolute path. Extracted for unit testing (same rationale as
* handleFileRequest — avoids CodeQL js/missing-rate-limiting false
* positives in tests).
*/
export const handleFsListRequest = async (
req: { query: any },
res: {
status: (code: number) => { json: (body: any) => void };
json: (body: any) => void;
},
): Promise<void> => {
try {
const raw = req.query.dir ?? '/';
const dir = assertString(raw, 'dir');
if (!path.isAbsolute(dir)) {
res.status(400).json({ error: '"dir" must be an absolute path' });
return;
}
// Skip the traversal guard for bare root directories (/ on Linux, C:\ on
// Windows) where path.normalize and path.resolve diverge on Windows.
const isRoot = dir === path.parse(dir).root;
if (!isRoot && path.normalize(dir) !== path.resolve(dir)) {
res.status(400).json({ error: '"dir" must not contain traversal sequences' });
return;
}
let dirents;
try {
dirents = await fs.readdir(dir, { withFileTypes: true });
} catch (err: unknown) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT' || code === 'ENOTDIR') {
res.status(404).json({ error: 'Directory not found' });
return;
}
if (code === 'EACCES' || code === 'EPERM') {
res.status(403).json({ error: 'Permission denied' });
return;
}
throw err;
}
const entries = dirents
.filter((d) => d.isDirectory())
.map((d) => ({ name: d.name }))
.sort((a, b) => a.name.localeCompare(b.name));
res.json({ entries });
} catch (err: unknown) {
if (err instanceof BadRequestError) {
res.status(err.status).json({ error: err.message });
return;
}
res.status(500).json({ error: 'Internal server error' });
}
};
/**
* Handle a GET /api/file request body. Extracted from createServer's route
* registration so it can be unit-tested without spinning up an HTTP server
@ -801,6 +744,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
const cleanupMcp = mountMCPEndpoints(app, backend);
const jobManager = new JobManager();
// Backstop: remove any upload staging dirs orphaned by a previous crash.
void sweepStaleUploads().catch(() => {});
// Shared repo lock — prevents concurrent analyze + embed on the same repo path,
// which would corrupt LadybugDB (analyze calls closeLbug + initLbug while embed has queries in flight).
const activeRepoPaths = new Set<string>();
@ -817,6 +763,149 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
activeRepoPaths.delete(repoPath);
};
// Launch the analyze worker for an already-resolved repo directory. Shared by
// the JSON /api/analyze route and the multipart /api/analyze/upload route so
// the lock + fork + auto-retry + IPC machinery lives in one place.
const launchAnalysisWorker = (
job: { id: string },
targetPath: string,
opts: {
force?: boolean;
embeddings?: boolean;
dropEmbeddings?: boolean;
registryName?: string;
},
): void => {
// Acquire shared repo lock (keyed on storagePath to match embed handler)
const analyzeLockKey = getStoragePath(targetPath);
const lockErr = acquireRepoLock(analyzeLockKey);
if (lockErr) {
jobManager.updateJob(job.id, { status: 'failed', error: lockErr });
return;
}
jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' });
// ── Worker fork with auto-retry ──────────────────────────────
const MAX_WORKER_RETRIES = 2;
const callerPath = fileURLToPath(import.meta.url);
const isDev = callerPath.endsWith('.ts');
const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js';
const workerPath = path.join(path.dirname(callerPath), workerFile);
const tsxHookArgs: string[] = isDev
? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href]
: [];
const forkWorker = () => {
const currentJob = jobManager.getJob(job.id);
if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed') return;
const child = fork(workerPath, [], {
execArgv: [...tsxHookArgs, '--max-old-space-size=8192'],
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
});
// Capture stderr for crash diagnostics
let stderrChunks = '';
child.stderr?.on('data', (chunk: Buffer) => {
stderrChunks += chunk.toString();
if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096);
});
child.on('message', (msg: any) => {
if (msg.type === 'progress') {
jobManager.updateJob(job.id, {
status: 'analyzing',
progress: { phase: msg.phase, percent: msg.percent, message: msg.message },
});
} else if (msg.type === 'complete') {
releaseRepoLock(analyzeLockKey);
// Reinitialize backend BEFORE marking complete — ensures the new
// repo is queryable when the client receives the SSE complete event.
backend
.init()
.then(() => {
jobManager.updateJob(job.id, {
status: 'complete',
repoName: msg.result.repoName,
});
})
.catch((err) => {
logger.error({ err }, 'backend.init() failed after analyze:');
jobManager.updateJob(job.id, {
status: 'failed',
error: 'Server failed to reload after analysis. Try again.',
});
});
} else if (msg.type === 'error') {
releaseRepoLock(analyzeLockKey);
jobManager.updateJob(job.id, {
status: 'failed',
error: msg.message,
});
}
});
child.on('error', (err) => {
releaseRepoLock(analyzeLockKey);
jobManager.updateJob(job.id, {
status: 'failed',
error: `Worker process error: ${err.message}`,
});
});
child.on('exit', (code) => {
const j = jobManager.getJob(job.id);
if (!j || j.status === 'complete' || j.status === 'failed') return;
// Worker crashed — attempt retry if under the limit
if (j.retryCount < MAX_WORKER_RETRIES) {
j.retryCount++;
const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s
const lastErr = stderrChunks.trim().split('\n').pop() || '';
logger.warn(
`Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` +
(lastErr ? `: ${lastErr}` : ''),
);
jobManager.updateJob(job.id, {
status: 'analyzing',
progress: {
phase: 'retrying',
percent: j.progress.percent,
message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`,
},
});
stderrChunks = '';
setTimeout(forkWorker, delay);
} else {
// Exhausted retries — permanent failure
releaseRepoLock(analyzeLockKey);
jobManager.updateJob(job.id, {
status: 'failed',
error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`,
});
}
});
// Register child for cancellation + timeout tracking
jobManager.registerChild(job.id, child);
// Send start command to child
child.send({
type: 'start',
repoPath: targetPath,
options: {
force: !!opts.force,
embeddings: !!opts.embeddings,
dropEmbeddings: !!opts.dropEmbeddings,
...(opts.registryName ? { registryName: opts.registryName } : {}),
},
});
};
forkWorker();
};
/**
* Maximum time the hold-queue will wait for an active analysis job to complete.
* Must stay in sync with the frontend's `fetchRepoInfo({ awaitAnalysis: true })` timeout.
@ -1054,6 +1143,14 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
}
}
// 2b. Delete the uploaded repo dir if entry.path lives under
// UPLOAD_ROOT. Drive this off entry.path (not a name-rederived dir) so
// a same-named clone is never affected.
const resolvedEntry = path.resolve(entry.path);
if (resolvedEntry === UPLOAD_ROOT || resolvedEntry.startsWith(UPLOAD_ROOT + path.sep)) {
await fs.rm(resolvedEntry, { recursive: true, force: true }).catch(() => {});
}
// 3. Unregister from the global registry
const { unregisterRepo } = await import('../storage/repo-manager.js');
await unregisterRepo(entry.path);
@ -1325,8 +1422,6 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
await handleFileRequest(req, res, entry.path);
});
app.get('/api/fs/list', createRouteLimiter(), (req, res) => handleFsListRequest(req, res));
// Grep — regex search across file contents in the indexed repo
// Uses filesystem-based search for memory efficiency (never loads all files into memory)
// Rate-limited (CodeQL js/missing-rate-limiting): scans every file in
@ -1490,229 +1585,127 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
// ── Analyze API ──────────────────────────────────────────────────────
// POST /api/analyze — start a new analysis job
app.post('/api/analyze', createRouteLimiter({ limit: 10 }), async (req, res) => {
try {
const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings } = req.body;
app.post(
'/api/analyze',
createRouteLimiter({ limit: 10 }),
requireLocalhostOrigin,
async (req, res) => {
try {
const { url: repoUrl, force, embeddings, dropEmbeddings } = req.body;
// `path` is canonicalized below (realpath), so keep it mutable.
let repoLocalPath: string | undefined = req.body.path;
// Input type validation
if (repoUrl !== undefined && typeof repoUrl !== 'string') {
res.status(400).json({ error: '"url" must be a string' });
return;
}
if (repoLocalPath !== undefined && typeof repoLocalPath !== 'string') {
res.status(400).json({ error: '"path" must be a string' });
return;
}
if (!repoUrl && !repoLocalPath) {
res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' });
return;
}
// Path validation: require absolute path, reject traversal (e.g. /tmp/../etc/passwd)
if (repoLocalPath) {
if (!path.isAbsolute(repoLocalPath)) {
res.status(400).json({ error: '"path" must be an absolute path' });
// Input type validation
if (repoUrl !== undefined && typeof repoUrl !== 'string') {
res.status(400).json({ error: '"url" must be a string' });
return;
}
if (path.normalize(repoLocalPath) !== path.resolve(repoLocalPath)) {
res.status(400).json({ error: '"path" must not contain traversal sequences' });
if (repoLocalPath !== undefined && typeof repoLocalPath !== 'string') {
res.status(400).json({ error: '"path" must be a string' });
return;
}
}
const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath });
if (!repoUrl && !repoLocalPath) {
res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' });
return;
}
// If job was already running (dedup), just return its id
if (job.status !== 'queued') {
res.status(202).json({ jobId: job.id, status: job.status });
return;
}
// Mark as active synchronously to prevent race with concurrent requests
jobManager.updateJob(job.id, { status: 'cloning' });
// Start async work — don't await
(async () => {
let targetPath = repoLocalPath;
try {
// Clone if URL provided
if (repoUrl && !repoLocalPath) {
const repoName = extractRepoName(repoUrl);
targetPath = getCloneDir(repoName);
jobManager.updateJob(job.id, {
status: 'cloning',
repoName,
progress: { phase: 'cloning', percent: 0, message: `Cloning ${repoUrl}...` },
});
await cloneOrPull(repoUrl, targetPath, (progress) => {
jobManager.updateJob(job.id, {
progress: { phase: progress.phase, percent: 5, message: progress.message },
});
});
}
if (!targetPath) {
throw new Error('No target path resolved');
}
// Acquire shared repo lock (keyed on storagePath to match embed handler)
const analyzeLockKey = getStoragePath(targetPath);
const lockErr = acquireRepoLock(analyzeLockKey);
if (lockErr) {
jobManager.updateJob(job.id, { status: 'failed', error: lockErr });
// Path validation. The previous `normalize !== resolve` guard was inert
// (both collapse `..` identically) and only false-rejected trailing
// slashes. Analyzing a local path the operator names is the tool's
// intended capability (same as the CLI); the cross-origin reach is what
// mattered, and that is closed by requireLocalhostOrigin above. Here we
// just require an absolute path that actually exists and is a directory,
// using its realpath-canonical form downstream.
if (repoLocalPath) {
if (!path.isAbsolute(repoLocalPath)) {
res.status(400).json({ error: '"path" must be an absolute path' });
return;
}
jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' });
// ── Worker fork with auto-retry ──────────────────────────────
//
// Forks a child process with 8GB heap. If the worker crashes
// (OOM, native addon segfault, etc.), it retries up to
// MAX_WORKER_RETRIES times with exponential backoff before
// marking the job as permanently failed.
//
// In dev mode (tsx), registers the tsx ESM hook via a file://
// URL so the child can compile TypeScript on-the-fly.
const MAX_WORKER_RETRIES = 2;
const callerPath = fileURLToPath(import.meta.url);
const isDev = callerPath.endsWith('.ts');
const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js';
const workerPath = path.join(path.dirname(callerPath), workerFile);
const tsxHookArgs: string[] = isDev
? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href]
: [];
const forkWorker = () => {
const currentJob = jobManager.getJob(job.id);
if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed')
try {
const resolved = await fs.realpath(repoLocalPath);
if (!(await fs.stat(resolved)).isDirectory()) {
res.status(400).json({ error: '"path" must be a directory' });
return;
const child = fork(workerPath, [], {
execArgv: [...tsxHookArgs, '--max-old-space-size=8192'],
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
});
// Capture stderr for crash diagnostics
let stderrChunks = '';
child.stderr?.on('data', (chunk: Buffer) => {
stderrChunks += chunk.toString();
if (stderrChunks.length > 4096) stderrChunks = stderrChunks.slice(-4096);
});
child.on('message', (msg: any) => {
if (msg.type === 'progress') {
jobManager.updateJob(job.id, {
status: 'analyzing',
progress: { phase: msg.phase, percent: msg.percent, message: msg.message },
});
} else if (msg.type === 'complete') {
releaseRepoLock(analyzeLockKey);
// Reinitialize backend BEFORE marking complete — ensures the new
// repo is queryable when the client receives the SSE complete event.
backend
.init()
.then(() => {
jobManager.updateJob(job.id, {
status: 'complete',
repoName: msg.result.repoName,
});
})
.catch((err) => {
logger.error({ err }, 'backend.init() failed after analyze:');
jobManager.updateJob(job.id, {
status: 'failed',
error: 'Server failed to reload after analysis. Try again.',
});
});
} else if (msg.type === 'error') {
releaseRepoLock(analyzeLockKey);
jobManager.updateJob(job.id, {
status: 'failed',
error: msg.message,
});
}
});
child.on('error', (err) => {
releaseRepoLock(analyzeLockKey);
jobManager.updateJob(job.id, {
status: 'failed',
error: `Worker process error: ${err.message}`,
});
});
child.on('exit', (code) => {
const j = jobManager.getJob(job.id);
if (!j || j.status === 'complete' || j.status === 'failed') return;
// Worker crashed — attempt retry if under the limit
if (j.retryCount < MAX_WORKER_RETRIES) {
j.retryCount++;
const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s
const lastErr = stderrChunks.trim().split('\n').pop() || '';
logger.warn(
`Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` +
(lastErr ? `: ${lastErr}` : ''),
);
jobManager.updateJob(job.id, {
status: 'analyzing',
progress: {
phase: 'retrying',
percent: j.progress.percent,
message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`,
},
});
stderrChunks = '';
setTimeout(forkWorker, delay);
} else {
// Exhausted retries — permanent failure
releaseRepoLock(analyzeLockKey);
jobManager.updateJob(job.id, {
status: 'failed',
error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`,
});
}
});
// Register child for cancellation + timeout tracking
jobManager.registerChild(job.id, child);
// Send start command to child
child.send({
type: 'start',
repoPath: targetPath,
options: {
force: !!force,
embeddings: !!embeddings,
dropEmbeddings: !!dropEmbeddings,
},
});
};
forkWorker();
} catch (err: any) {
if (targetPath) releaseRepoLock(getStoragePath(targetPath));
jobManager.updateJob(job.id, {
status: 'failed',
error: err.message || 'Analysis failed',
});
}
repoLocalPath = resolved;
} catch {
res.status(404).json({ error: '"path" does not exist or is not accessible' });
return;
}
}
})();
res.status(202).json({ jobId: job.id, status: job.status });
} catch (err: any) {
if (err.message?.includes('already in progress')) {
res.status(409).json({ error: err.message });
} else {
res.status(500).json({ error: err.message || 'Failed to start analysis' });
const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath });
// If job was already running (dedup), just return its id
if (job.status !== 'queued') {
res.status(202).json({ jobId: job.id, status: job.status });
return;
}
// Mark as active synchronously to prevent race with concurrent requests
jobManager.updateJob(job.id, { status: 'cloning' });
// Start async work — don't await
(async () => {
let targetPath = repoLocalPath;
try {
// Clone if URL provided
if (repoUrl && !repoLocalPath) {
const repoName = extractRepoName(repoUrl);
targetPath = getCloneDir(repoName);
jobManager.updateJob(job.id, {
status: 'cloning',
repoName,
progress: { phase: 'cloning', percent: 0, message: `Cloning ${repoUrl}...` },
});
await cloneOrPull(repoUrl, targetPath, (progress) => {
jobManager.updateJob(job.id, {
progress: { phase: progress.phase, percent: 5, message: progress.message },
});
});
}
if (!targetPath) {
throw new Error('No target path resolved');
}
launchAnalysisWorker(job, targetPath, { force, embeddings, dropEmbeddings });
} catch (err: any) {
if (targetPath) releaseRepoLock(getStoragePath(targetPath));
jobManager.updateJob(job.id, {
status: 'failed',
error: err.message || 'Analysis failed',
});
}
})();
res.status(202).json({ jobId: job.id, status: job.status });
} catch (err: any) {
if (err.message?.includes('already in progress')) {
res.status(409).json({ error: err.message });
} else {
res.status(500).json({ error: err.message || 'Failed to start analysis' });
}
}
}
});
},
);
// POST /api/analyze/upload — analyze a browser folder upload.
// Securely ingests the multipart upload into a sandbox, promotes it to a
// persistent dir, and analyzes it via the shared job/worker machinery.
// localhost-only (no cross-origin write reach) + conservative rate limit.
app.post(
'/api/analyze/upload',
createRouteLimiter({ limit: 5 }),
requireLocalhostOrigin,
createAnalyzeUploadHandler({
createJob: (params) => jobManager.createJob(params),
launch: (job, targetPath, opts) => launchAnalysisWorker(job, targetPath, opts),
}),
);
// GET /api/analyze/:jobId — poll job status
app.get('/api/analyze/:jobId', (req, res) => {

304
gitnexus/src/server/upload-ingest.ts Normal file
View file

@ -0,0 +1,304 @@
/**
* Secure ingestion of a browser folder upload (multipart/form-data).
*
* Replaces the path-injection-prone GET /api/fs/list directory listing. The
* client streams the selected files plus a JSON `manifest` of their
* webkitRelativePaths; we write each into an mkdtemp staging dir under
* UPLOAD_ROOT with PROVABLE containment (resolve-then-contain), hard resource
* caps, and guaranteed cleanup on every failure/abort path. No client value
* ever reaches a filesystem READ — the server only writes into a sandbox it
* created, then hands that sandbox to the analysis pipeline.
*
* Security references: CodeQL js/path-injection (resolve + startsWith(root+sep)),
* OWASP File Upload / Path Traversal.
*/
import path from 'path';
import fs from 'fs';
import fsp from 'fs/promises';
import type { IncomingMessage } from 'http';
import busboy from 'busboy';
import { UPLOAD_ROOT, STAGING_PREFIX } from './upload-paths.js';
import { BadRequestError } from './validation.js';
export interface IngestLimits {
/** Aggregate bytes across all files (busboy has no aggregate limit). */
maxTotalBytes: number;
/** Per-file byte cap. */
maxFileBytes: number;
/** Maximum number of files. */
maxFiles: number;
/** Maximum multipart parts (files + fields). */
maxParts: number;
/** Maximum directories created (inode-exhaustion guard). */
maxDirs: number;
/** Maximum size of the manifest field. */
maxFieldBytes: number;
}
export const DEFAULT_INGEST_LIMITS: IngestLimits = {
maxTotalBytes: 250 * 1024 * 1024,
maxFileBytes: 25 * 1024 * 1024,
maxFiles: 20000,
maxParts: 20100,
maxDirs: 50000,
maxFieldBytes: 2 * 1024 * 1024,
};
const MAX_PATH_DEPTH = 64;
const MAX_PATH_LENGTH = 4096;
export interface IngestResult {
/** Absolute path to the populated staging directory (realpath-canonical). */
stageRoot: string;
fileCount: number;
totalBytes: number;
/** First path segment shared by the uploaded tree (the picked folder). */
topLevelName: string;
}
/**
* Resolve a client-provided relative path to an absolute destination PROVABLY
* contained within `stageRoot`. Throws BadRequestError on any unsafe input.
* This is the load-bearing path-traversal-on-write control; keep it pure and
* unit-tested.
*/
export function resolveContainedDest(stageRoot: string, rel: unknown): string {
if (typeof rel !== 'string' || rel.length === 0) {
throw new BadRequestError('Invalid upload path');
}
if (rel.length > MAX_PATH_LENGTH) {
throw new BadRequestError('Upload path too long');
}
// webkitRelativePath is always relative; a leading slash is absolute/hostile.
if (rel.startsWith('/')) {
throw new BadRequestError('Invalid upload path');
}
// Browsers emit forward slashes only; a NUL byte or backslash is hostile.
if (rel.includes('\u0000') || rel.includes('\\')) {
throw new BadRequestError('Invalid upload path');
}
const rawSegments = rel.split('/').filter((s) => s.length > 0);
if (rawSegments.length === 0 || rawSegments.length > MAX_PATH_DEPTH) {
throw new BadRequestError('Invalid upload path');
}
const segments: string[] = [];
for (const seg of rawSegments) {
// Normalize so NFC/NFD variants don't collide silently on case/unicode
// -folding filesystems (macOS/Windows).
const s = seg.normalize('NFC');
if (s === '.' || s === '..') {
throw new BadRequestError('Upload path must not contain traversal segments');
}
segments.push(s);
}
const dest = path.resolve(stageRoot, segments.join(path.sep));
// Suffix path.sep so a sibling prefix (/sandbox-evil vs /sandbox) can't pass.
if (dest !== stageRoot && !dest.startsWith(stageRoot + path.sep)) {
throw new BadRequestError('Upload path escapes the sandbox');
}
return dest;
}
interface DirState {
dirCount: number;
limits: IngestLimits;
}
/**
* Create the parent directories of `destFile` one segment at a time, asserting
* after each `mkdir` that the segment is a real directory (not a symlink
* swapped in mid-stream) still inside `stageRoot`. Counts created dirs against
* the inode-exhaustion cap.
*/
function mkdirContained(stageRoot: string, destFile: string, state: DirState): void {
const parent = path.dirname(destFile);
const relParent = path.relative(stageRoot, parent);
if (relParent === '' || relParent === '.') return;
const segs = relParent.split(path.sep).filter(Boolean);
let cur = stageRoot;
for (const seg of segs) {
cur = path.join(cur, seg);
let made = false;
try {
fs.mkdirSync(cur);
made = true;
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err;
}
const st = fs.lstatSync(cur);
if (st.isSymbolicLink() || !st.isDirectory()) {
throw new BadRequestError('Upload path escapes the sandbox');
}
if (made) {
state.dirCount++;
if (state.dirCount > state.limits.maxDirs) {
throw new BadRequestError('Too many directories in upload', 413);
}
}
}
}
/**
* Parse and securely write a multipart folder upload into a fresh staging
* directory under UPLOAD_ROOT. Resolves with the populated staging dir, or
* rejects with a BadRequestError (status 400/413) after removing the staging
* dir. The caller owns promotion + cleanup of the returned `stageRoot`.
*/
export async function ingestUpload(
req: IncomingMessage,
limitsOverride?: Partial<IngestLimits>,
): Promise<IngestResult> {
const limits = { ...DEFAULT_INGEST_LIMITS, ...limitsOverride };
await fsp.mkdir(UPLOAD_ROOT, { recursive: true });
// mkdtemp creates the dir mode 0o700 (owner-only); realpath canonicalizes
// the root so the containment prefix check is exact.
const stageRoot = await fsp.realpath(await fsp.mkdtemp(path.join(UPLOAD_ROOT, STAGING_PREFIX)));
let cleaned = false;
const cleanup = async (): Promise<void> => {
if (cleaned) return;
cleaned = true;
await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => {});
};
return new Promise<IngestResult>((resolve, reject) => {
let settled = false;
let manifest: string[] | null = null;
let fileIndex = 0;
let fileCount = 0;
let totalBytes = 0;
let topLevelName = '';
const dirState: DirState = { dirCount: 0, limits };
const writePromises: Promise<void>[] = [];
const bb = busboy({
headers: req.headers,
limits: {
fileSize: limits.maxFileBytes,
files: limits.maxFiles,
parts: limits.maxParts,
fields: 10,
fieldNameSize: 200,
fieldSize: limits.maxFieldBytes,
headerPairs: 2000,
},
});
const fail = (err: Error): void => {
if (settled) return;
settled = true;
try {
req.unpipe(bb);
} catch {
/* ignore */
}
try {
req.resume(); // drain remaining body so the socket isn't left hanging
} catch {
/* ignore */
}
void cleanup().finally(() => reject(err));
};
bb.on('field', (name: string, val: string) => {
if (name !== 'manifest') return;
try {
const parsed = JSON.parse(val);
if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === 'string')) {
return fail(new BadRequestError('Invalid manifest'));
}
manifest = parsed as string[];
} catch {
fail(new BadRequestError('Invalid manifest'));
}
});
bb.on('file', (_name: string, stream: NodeJS.ReadableStream, _info: unknown) => {
if (settled) {
stream.resume();
return;
}
if (manifest === null) {
// The manifest field MUST arrive before any file part.
stream.resume();
return fail(new BadRequestError('Manifest must precede file parts'));
}
const idx = fileIndex++;
const rel = manifest[idx];
let dest: string;
try {
dest = resolveContainedDest(stageRoot, rel);
if (!topLevelName) {
// The on-disk top folder uses the NFC-normalized first segment
// (matching how each path segment is written); the promote step
// renames stageRoot/<topLevelName> into place.
topLevelName = (String(rel).split('/').filter(Boolean)[0] ?? '').normalize('NFC');
}
mkdirContained(stageRoot, dest, dirState);
} catch (err) {
stream.resume();
return fail(err as Error);
}
fileCount++;
const ws = fs.createWriteStream(dest, { flags: 'wx' });
const p = new Promise<void>((res, rej) => {
stream.on('data', (chunk: Buffer) => {
totalBytes += chunk.length;
if (totalBytes > limits.maxTotalBytes) {
stream.unpipe(ws);
ws.destroy();
rej(new BadRequestError('Upload exceeds total size limit', 413));
}
});
stream.on('limit', () => {
ws.destroy();
rej(new BadRequestError('File exceeds size limit', 413));
});
stream.on('error', rej);
ws.on('error', rej);
ws.on('finish', () => res());
stream.pipe(ws);
});
writePromises.push(p);
p.catch(fail);
});
bb.on('filesLimit', () => fail(new BadRequestError('Too many files in upload', 413)));
bb.on('partsLimit', () => fail(new BadRequestError('Too many parts in upload', 413)));
bb.on('fieldsLimit', () => fail(new BadRequestError('Too many fields in upload')));
bb.on('error', (err: unknown) =>
fail(err instanceof Error ? err : new BadRequestError('Upload parse error')),
);
bb.on('close', () => {
if (settled) return;
Promise.all(writePromises)
.then(() => {
if (settled) return;
if (manifest === null) return fail(new BadRequestError('Missing manifest'));
if (fileCount === 0) return fail(new BadRequestError('Empty upload'));
if (fileCount !== manifest.length) {
return fail(new BadRequestError('Manifest/file count mismatch'));
}
if (!topLevelName) {
return fail(new BadRequestError('Could not determine upload folder name'));
}
settled = true;
resolve({ stageRoot, fileCount, totalBytes, topLevelName });
})
.catch(() => {
/* a write rejected → fail() already invoked via p.catch */
});
});
req.on('aborted', () => {
if (!settled) fail(new BadRequestError('Upload aborted'));
});
req.on('error', (err: unknown) =>
fail(err instanceof Error ? err : new BadRequestError('Request error')),
);
req.pipe(bb);
});
}

62
gitnexus/src/server/upload-paths.ts Normal file
View file

@ -0,0 +1,62 @@
/**
* Upload working-directory paths.
*
* Browser folder uploads are written into ~/.gitnexus/uploads/{name}/ — a
* sibling of the clone root (git-clone.ts CLONE_ROOT) — so an uploaded repo
* persists and behaves like a cloned one (the graph UI's /api/file reads its
* files after analysis, and DELETE /api/repo removes it). Staging happens in
* an mkdtemp dir *under* UPLOAD_ROOT so the promote rename stays on one
* filesystem and remains atomic (a rename from os.tmpdir() could trip EXDEV —
* the exact Docker case this feature targets; see bridge-db.ts for the same
* anchored-staging pattern).
*/
import path from 'path';
import os from 'os';
import { sanitizeRepoName } from '../storage/git.js';
/** Root directory for all uploaded repositories. Targets must resolve inside this. */
export const UPLOAD_ROOT = path.resolve(path.join(os.homedir(), '.gitnexus', 'uploads'));
/** Prefix for per-upload staging directories created under UPLOAD_ROOT. */
export const STAGING_PREFIX = '.staging-';
// Filesystem-safe repo name: alphanumerics plus `. _ -`. Mirrors
// git-clone.ts REPO_NAME_PATTERN so getUploadDir(name) cannot escape
// UPLOAD_ROOT regardless of how the caller derived the name.
const UPLOAD_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/;
/**
* Get the upload target directory for a repo name.
*
* Re-validates at the boundary (callers may derive the name from an untrusted
* manifest). Rejects `.`, `..`, the `'unknown'` sentinel that sanitizeRepoName
* emits for un-nameable inputs, names beginning with `.` (which would collide
* with the `.staging-` prefix), and anything outside the safe charset.
*/
export function getUploadDir(repoName: string): string {
if (
!repoName ||
repoName === '.' ||
repoName === '..' ||
repoName === 'unknown' ||
repoName.startsWith('.') ||
!UPLOAD_NAME_PATTERN.test(repoName)
) {
throw new Error('Invalid repository name');
}
return path.join(UPLOAD_ROOT, repoName);
}
/**
* Derive a filesystem-safe upload directory name from the manifest's
* top-level folder. Returns null when the name is un-nameable (so the caller
* rejects with 400 rather than colliding everyone on `UPLOAD_ROOT/unknown`).
*/
export function deriveUploadName(topLevelName: string): string | null {
const safe = sanitizeRepoName(topLevelName);
if (safe === 'unknown' || safe === '.' || safe === '..' || safe.startsWith('.')) {
return null;
}
return safe;
}

52
gitnexus/src/server/upload-sweep.ts Normal file
View file

@ -0,0 +1,52 @@
/**
* Backstop cleanup for abandoned upload staging directories.
*
* A crashed/killed process can leave a `.staging-*` directory under
* UPLOAD_ROOT (the normal path removes it on success/failure/abort). This
* sweep, run once at server startup, removes staging dirs older than a
* threshold. Promoted upload dirs are persistent registered repos (like
* clones) and are NOT touched here — they are removed via DELETE /api/repo.
*/
import path from 'path';
import fsp from 'fs/promises';
import { UPLOAD_ROOT, STAGING_PREFIX } from './upload-paths.js';
export interface SweepOptions {
/** Remove staging dirs older than this (default 6h). */
maxAgeMs?: number;
/** Override the root to sweep (defaults to UPLOAD_ROOT; for tests). */
root?: string;
/** Clock injection for tests. */
now?: number;
}
export async function sweepStaleUploads(opts: SweepOptions = {}): Promise<{ removed: string[] }> {
const maxAgeMs = opts.maxAgeMs ?? 6 * 60 * 60 * 1000;
const root = opts.root ?? UPLOAD_ROOT;
const now = opts.now ?? Date.now();
const removed: string[] = [];
let entries;
try {
entries = await fsp.readdir(root, { withFileTypes: true });
} catch {
return { removed }; // root does not exist yet — nothing to sweep
}
for (const entry of entries) {
if (!entry.isDirectory() || !entry.name.startsWith(STAGING_PREFIX)) continue;
const full = path.join(root, entry.name);
try {
const st = await fsp.stat(full);
if (now - st.mtimeMs > maxAgeMs) {
await fsp.rm(full, { recursive: true, force: true }).catch(() => {});
removed.push(full);
}
} catch {
/* stat race — skip */
}
}
return { removed };
}

221
gitnexus/test/unit/api-analyze-upload.test.ts Normal file
View file

@ -0,0 +1,221 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import path from 'node:path';
import fs from 'node:fs/promises';
import { Readable } from 'node:stream';
import type { IncomingMessage } from 'node:http';
import {
createAnalyzeUploadHandler,
requireLocalhostOrigin,
} from '../../src/server/analyze-upload.js';
const BOUNDARY = '----gitnexusuploadtest';
function multipart(
parts: Array<{ name: string; value?: string; filename?: string; data?: Buffer }>,
): { body: Buffer; headers: Record<string, string> } {
const chunks: Buffer[] = [];
for (const p of parts) {
chunks.push(Buffer.from(`--${BOUNDARY}\r\n`));
if (p.filename !== undefined) {
chunks.push(
Buffer.from(
`Content-Disposition: form-data; name="${p.name}"; filename="${p.filename}"\r\n` +
`Content-Type: application/octet-stream\r\n\r\n`,
),
);
chunks.push(p.data ?? Buffer.alloc(0));
chunks.push(Buffer.from('\r\n'));
} else {
chunks.push(Buffer.from(`Content-Disposition: form-data; name="${p.name}"\r\n\r\n`));
chunks.push(Buffer.from(p.value ?? ''));
chunks.push(Buffer.from('\r\n'));
}
}
chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`));
return {
body: Buffer.concat(chunks),
headers: { 'content-type': `multipart/form-data; boundary=${BOUNDARY}` },
};
}
function mockReq(parts: Parameters<typeof multipart>[0]): IncomingMessage {
const { body, headers } = multipart(parts);
const r = Readable.from([body]) as unknown as IncomingMessage & { headers: typeof headers };
r.headers = headers;
return r;
}
interface MockRes {
statusCode: number;
body: unknown;
status: (c: number) => MockRes;
json: (b: unknown) => MockRes;
}
function mockRes(): MockRes {
const res = { statusCode: 0, body: undefined as unknown } as MockRes;
res.status = (c: number) => {
res.statusCode = c;
return res;
};
res.json = (b: unknown) => {
res.body = b;
return res;
};
return res;
}
// Track promoted upload dirs created by the real ingest+promote so we clean up.
const promoted: string[] = [];
afterEach(async () => {
while (promoted.length) {
await fs.rm(promoted.pop()!, { recursive: true, force: true }).catch(() => {});
}
});
function uniqueTop(): string {
return `uptest_${Math.floor(Math.random() * 1e9).toString(36)}`;
}
describe('createAnalyzeUploadHandler', () => {
it('ingests, promotes the inner folder, and launches analysis (202)', async () => {
const top = uniqueTop();
const createJob = vi.fn(() => ({ id: 'job-1', status: 'queued' }));
const launch = vi.fn((_j, dir: string) => promoted.push(dir));
const handler = createAnalyzeUploadHandler({ createJob, launch });
const res = mockRes();
await handler(
mockReq([
{ name: 'manifest', value: JSON.stringify([`${top}/a.js`, `${top}/sub/b.js`]) },
{ name: 'files', filename: 'blob', data: Buffer.from('alpha') },
{ name: 'files', filename: 'blob', data: Buffer.from('beta') },
]) as never,
res as never,
);
expect(res.statusCode).toBe(202);
expect((res.body as { jobId: string }).jobId).toBe('job-1');
expect(createJob).toHaveBeenCalledOnce();
expect(launch).toHaveBeenCalledOnce();
const dir = launch.mock.calls[0][1] as string;
const opts = launch.mock.calls[0][2] as { registryName: string };
// Inner folder promoted: contents live directly under the upload dir.
expect(await fs.readFile(path.join(dir, 'a.js'), 'utf8')).toBe('alpha');
expect(await fs.readFile(path.join(dir, 'sub', 'b.js'), 'utf8')).toBe('beta');
expect(opts.registryName).toBe(top);
expect(createJob.mock.calls[0][0].repoPath).toBe(dir);
});
it('maps a busy job (createJob throws "already in progress") to 409 and promotes nothing', async () => {
const top = uniqueTop();
const createJob = vi.fn(() => {
throw new Error('Analysis already in progress for another repository');
});
const launch = vi.fn((_j, dir: string) => promoted.push(dir));
const handler = createAnalyzeUploadHandler({ createJob, launch });
const res = mockRes();
await handler(
mockReq([
{ name: 'manifest', value: JSON.stringify([`${top}/a.js`]) },
{ name: 'files', filename: 'blob', data: Buffer.from('x') },
]) as never,
res as never,
);
expect(res.statusCode).toBe(409);
expect(launch).not.toHaveBeenCalled();
// Nothing promoted onto disk.
const { UPLOAD_ROOT } = await import('../../src/server/upload-paths.js');
await expect(fs.access(path.join(UPLOAD_ROOT, top))).rejects.toBeTruthy();
});
it('rejects a traversal path in the manifest (400) without launching', async () => {
const createJob = vi.fn(() => ({ id: 'j', status: 'queued' }));
const launch = vi.fn();
const handler = createAnalyzeUploadHandler({ createJob, launch });
const res = mockRes();
await handler(
mockReq([
{ name: 'manifest', value: JSON.stringify(['../escape.js']) },
{ name: 'files', filename: 'blob', data: Buffer.from('x') },
]) as never,
res as never,
);
expect(res.statusCode).toBe(400);
expect(createJob).not.toHaveBeenCalled();
expect(launch).not.toHaveBeenCalled();
});
it('rejects an un-nameable top folder (Windows-reserved → 400)', async () => {
const createJob = vi.fn(() => ({ id: 'j', status: 'queued' }));
const launch = vi.fn();
const handler = createAnalyzeUploadHandler({ createJob, launch });
const res = mockRes();
await handler(
mockReq([
{ name: 'manifest', value: JSON.stringify(['CON/a.js']) },
{ name: 'files', filename: 'blob', data: Buffer.from('x') },
]) as never,
res as never,
);
expect(res.statusCode).toBe(400);
expect(launch).not.toHaveBeenCalled();
});
it('strips a crafted .gitnexus index from the promoted upload', async () => {
const top = uniqueTop();
const createJob = vi.fn(() => ({ id: 'job-x', status: 'queued' }));
const launch = vi.fn((_j, dir: string) => promoted.push(dir));
const handler = createAnalyzeUploadHandler({ createJob, launch });
const res = mockRes();
await handler(
mockReq([
{ name: 'manifest', value: JSON.stringify([`${top}/.gitnexus/meta.json`, `${top}/a.js`]) },
{ name: 'files', filename: 'blob', data: Buffer.from('{"evil":true}') },
{ name: 'files', filename: 'blob', data: Buffer.from('real') },
]) as never,
res as never,
);
expect(res.statusCode).toBe(202);
const dir = launch.mock.calls[0][1] as string;
await expect(fs.access(path.join(dir, '.gitnexus'))).rejects.toBeTruthy();
expect(await fs.readFile(path.join(dir, 'a.js'), 'utf8')).toBe('real');
});
});
describe('requireLocalhostOrigin', () => {
function call(origin: string | undefined): { passed: boolean; status: number } {
let passed = false;
let status = 0;
const req = { headers: origin === undefined ? {} : { origin } } as never;
const res = {
status: (c: number) => {
status = c;
return { json: () => {} };
},
} as never;
requireLocalhostOrigin(req, res, () => {
passed = true;
});
return { passed, status };
}
it('passes localhost / 127.0.0.1 / no-origin', () => {
expect(call('http://localhost:5173').passed).toBe(true);
expect(call('http://127.0.0.1:4747').passed).toBe(true);
expect(call(undefined).passed).toBe(true);
});
it('rejects a public/cross origin with 403', () => {
const r = call('https://gitnexus.vercel.app');
expect(r.passed).toBe(false);
expect(r.status).toBe(403);
});
});

101
gitnexus/test/unit/api-fs-list.test.ts
View file

@ -1,101 +0,0 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import path from 'node:path';
import fs from 'node:fs/promises';
import os from 'node:os';
import { handleFsListRequest } from '../../src/server/api.js';
let tmpRoot: string;
beforeAll(async () => {
tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-api-fs-list-test-'));
await fs.mkdir(path.join(tmpRoot, 'alpha'));
await fs.mkdir(path.join(tmpRoot, 'beta'));
await fs.writeFile(path.join(tmpRoot, 'file.txt'), 'hello\n', 'utf-8');
});
afterAll(async () => {
await fs.rm(tmpRoot, { recursive: true, force: true });
});
const invoke = async (query: Record<string, unknown>): Promise<{ status: number; body: any }> => {
let capturedStatus = 200;
let capturedBody: any = undefined;
const res = {
status(code: number) {
capturedStatus = code;
return this;
},
json(body: any) {
capturedBody = body;
},
};
await handleFsListRequest({ query }, res);
return { status: capturedStatus, body: capturedBody };
};
describe('GET /api/fs/list — handleFsListRequest', () => {
it('lists only subdirectories, sorted alphabetically', async () => {
const { status, body } = await invoke({ dir: tmpRoot });
expect(status).toBe(200);
expect(body.entries).toEqual([{ name: 'alpha' }, { name: 'beta' }]);
});
it('excludes files from the listing', async () => {
const { body } = await invoke({ dir: tmpRoot });
const names = body.entries.map((e: { name: string }) => e.name);
expect(names).not.toContain('file.txt');
});
it('returns empty entries for an empty directory', async () => {
const { status, body } = await invoke({ dir: path.join(tmpRoot, 'alpha') });
expect(status).toBe(200);
expect(body.entries).toEqual([]);
});
it('defaults to / when dir is omitted (linux server)', async () => {
const { status } = await invoke({});
// On Linux root / is the filesystem root; on Windows path.normalize('/')
// normalizes to \ while path.resolve('/') resolves to the CWD drive root.
// Our guard skips the traversal check for bare root paths.
expect(status).toBe(200);
});
it('returns 400 for a relative path', async () => {
const { status, body } = await invoke({ dir: 'relative/path' });
expect(status).toBe(400);
expect(body.error).toMatch(/absolute path/);
});
it('returns 400 for a non-canonical absolute path', async () => {
// On Windows, `/foo` is absolute but normalize(\foo) !== resolve(D:\foo)
// On Linux, construct a path with redundant separators that stays absolute
const nonCanonical = process.platform === 'win32' ? '/nonexistent' : `${tmpRoot}/./alpha`;
const { status } = await invoke({ dir: nonCanonical });
if (process.platform === 'win32') {
expect(status).toBe(400);
} else {
// On Linux, normalize and resolve agree for all absolute paths
expect(status).toBe(200);
}
});
it('returns 404 for a non-existent directory', async () => {
const { status, body } = await invoke({ dir: path.join(tmpRoot, 'nonexistent') });
expect(status).toBe(404);
expect(body.error).toMatch(/not found/i);
});
it('returns 400 when dir is an array (type confusion)', async () => {
const { status, body } = await invoke({ dir: [tmpRoot, '/etc'] });
expect(status).toBe(400);
expect(body.error).toMatch(/single string/);
});
it('route source is wired with createRouteLimiter', async () => {
const source = await fs.readFile(
path.join(import.meta.dirname, '..', '..', 'src', 'server', 'api.ts'),
'utf-8',
);
expect(source).toMatch(/app\.get\('\/api\/fs\/list',\s*createRouteLimiter\(\)/);
});
});

177
gitnexus/test/unit/upload-ingest.test.ts Normal file
View file

@ -0,0 +1,177 @@
import { afterEach, describe, expect, it } from 'vitest';
import path from 'node:path';
import fs from 'node:fs/promises';
import os from 'node:os';
import { Readable } from 'node:stream';
import type { IncomingMessage } from 'node:http';
import {
resolveContainedDest,
ingestUpload,
DEFAULT_INGEST_LIMITS,
} from '../../src/server/upload-ingest.js';
// ── resolveContainedDest (pure sanitizer — the load-bearing control) ──────────
describe('resolveContainedDest', () => {
const ROOT = path.resolve('/tmp/gitnexus-sandbox');
it('contains a legitimate nested path under the root', () => {
expect(resolveContainedDest(ROOT, 'myrepo/src/index.js')).toBe(
path.join(ROOT, 'myrepo', 'src', 'index.js'),
);
});
it('allows spaces in names (regression: NUL check must not reject spaces)', () => {
expect(resolveContainedDest(ROOT, 'my repo/a b.js')).toBe(path.join(ROOT, 'my repo', 'a b.js'));
});
it.each([
['parent traversal', '../../etc/passwd'],
['mid traversal', 'a/../../b'],
['dot-dot segment', 'a/../b'],
['dot segment', 'a/./b'],
['absolute path', '/etc/shadow'],
['backslash', 'a\\b'],
['empty', ''],
])('rejects %s', (_label, rel) => {
expect(() => resolveContainedDest(ROOT, rel)).toThrow();
});
it('rejects a NUL byte', () => {
expect(() => resolveContainedDest(ROOT, `a${String.fromCharCode(0)}b/x.js`)).toThrow();
});
it('rejects non-string input', () => {
// @ts-expect-error testing runtime guard
expect(() => resolveContainedDest(ROOT, ['a', 'b'])).toThrow();
});
it('rejects an over-deep path (>64 segments)', () => {
const deep = Array.from({ length: 70 }, (_, i) => `d${i}`).join('/') + '/f.js';
expect(() => resolveContainedDest(ROOT, deep)).toThrow();
});
it('rejects an over-long path (>4096 chars)', () => {
const long = 'a/'.repeat(2100) + 'f.js';
expect(() => resolveContainedDest(ROOT, long)).toThrow();
});
it('rejects a sibling-prefix escape (root + sep, not bare startsWith)', () => {
// A rel that would resolve to a sibling dir sharing the root's string prefix.
expect(() => resolveContainedDest(ROOT, '../gitnexus-sandbox-evil/x.js')).toThrow();
});
});
// ── ingestUpload (multipart streaming + containment + caps + cleanup) ──────────
const BOUNDARY = '----gitnexustestboundary';
function multipart(
parts: Array<{ name: string; value?: string; filename?: string; data?: Buffer }>,
): { body: Buffer; headers: Record<string, string> } {
const chunks: Buffer[] = [];
for (const p of parts) {
chunks.push(Buffer.from(`--${BOUNDARY}\r\n`));
if (p.filename !== undefined) {
chunks.push(
Buffer.from(
`Content-Disposition: form-data; name="${p.name}"; filename="${p.filename}"\r\n` +
`Content-Type: application/octet-stream\r\n\r\n`,
),
);
chunks.push(p.data ?? Buffer.alloc(0));
chunks.push(Buffer.from('\r\n'));
} else {
chunks.push(Buffer.from(`Content-Disposition: form-data; name="${p.name}"\r\n\r\n`));
chunks.push(Buffer.from(p.value ?? ''));
chunks.push(Buffer.from('\r\n'));
}
}
chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`));
return {
body: Buffer.concat(chunks),
headers: { 'content-type': `multipart/form-data; boundary=${BOUNDARY}` },
};
}
function mockReq(body: Buffer, headers: Record<string, string>): IncomingMessage {
const r = Readable.from([body]) as unknown as IncomingMessage & { headers: typeof headers };
r.headers = headers;
return r;
}
const staged: string[] = [];
afterEach(async () => {
while (staged.length) {
await fs.rm(staged.pop()!, { recursive: true, force: true }).catch(() => {});
}
});
describe('ingestUpload', () => {
it('writes a manifest-described tree into the sandbox and returns its shape', async () => {
const { body, headers } = multipart([
{ name: 'manifest', value: JSON.stringify(['myrepo/a.js', 'myrepo/sub/b.js']) },
{ name: 'files', filename: 'blob', data: Buffer.from('alpha') },
{ name: 'files', filename: 'blob', data: Buffer.from('beta') },
]);
const result = await ingestUpload(mockReq(body, headers));
staged.push(result.stageRoot);
expect(result.fileCount).toBe(2);
expect(result.topLevelName).toBe('myrepo');
expect(result.totalBytes).toBe('alpha'.length + 'beta'.length);
expect(await fs.readFile(path.join(result.stageRoot, 'myrepo', 'a.js'), 'utf8')).toBe('alpha');
expect(await fs.readFile(path.join(result.stageRoot, 'myrepo', 'sub', 'b.js'), 'utf8')).toBe(
'beta',
);
});
it('rejects a traversal path in the manifest and removes the staging dir', async () => {
const { body, headers } = multipart([
{ name: 'manifest', value: JSON.stringify(['../escape.js']) },
{ name: 'files', filename: 'blob', data: Buffer.from('x') },
]);
const before = await countStaging();
await expect(ingestUpload(mockReq(body, headers))).rejects.toMatchObject({ status: 400 });
// No new staging dir should survive.
expect(await countStaging()).toBeLessThanOrEqual(before);
});
it('rejects a file part that arrives before the manifest', async () => {
const { body, headers } = multipart([
{ name: 'files', filename: 'blob', data: Buffer.from('x') },
{ name: 'manifest', value: JSON.stringify(['a/x.js']) },
]);
await expect(ingestUpload(mockReq(body, headers))).rejects.toThrow(/Manifest must precede/);
});
it('rejects when total bytes exceed the cap (413)', async () => {
const { body, headers } = multipart([
{ name: 'manifest', value: JSON.stringify(['r/big.bin']) },
{ name: 'files', filename: 'blob', data: Buffer.alloc(64, 1) },
]);
await expect(ingestUpload(mockReq(body, headers), { maxTotalBytes: 16 })).rejects.toMatchObject(
{ status: 413 },
);
});
it('rejects an empty upload (0 files)', async () => {
const { body, headers } = multipart([{ name: 'manifest', value: JSON.stringify([]) }]);
await expect(ingestUpload(mockReq(body, headers))).rejects.toThrow();
});
it('exposes sane default caps', () => {
expect(DEFAULT_INGEST_LIMITS.maxTotalBytes).toBe(250 * 1024 * 1024);
expect(DEFAULT_INGEST_LIMITS.maxFiles).toBe(20000);
});
});
async function countStaging(): Promise<number> {
const root = path.resolve(path.join(os.homedir(), '.gitnexus', 'uploads'));
try {
const entries = await fs.readdir(root);
return entries.filter((e) => e.startsWith('.staging-')).length;
} catch {
return 0;
}
}

43
gitnexus/test/unit/upload-sweep.test.ts Normal file
View file

@ -0,0 +1,43 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import path from 'node:path';
import fs from 'node:fs/promises';
import os from 'node:os';
import { sweepStaleUploads } from '../../src/server/upload-sweep.js';
let root: string;
beforeEach(async () => {
root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-sweep-test-'));
});
afterEach(async () => {
await fs.rm(root, { recursive: true, force: true }).catch(() => {});
});
describe('sweepStaleUploads', () => {
it('removes stale staging dirs but keeps recent ones and non-staging dirs', async () => {
await fs.mkdir(path.join(root, '.staging-old'));
await fs.mkdir(path.join(root, '.staging-new'));
await fs.mkdir(path.join(root, 'myrepo')); // a promoted (persistent) upload dir
const now = 1_000_000_000_000;
// Age the "old" staging dir well past the threshold.
const old = new Date(now - 10 * 60 * 60 * 1000);
await fs.utimes(path.join(root, '.staging-old'), old, old);
const recent = new Date(now - 60 * 1000);
await fs.utimes(path.join(root, '.staging-new'), recent, recent);
const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 });
expect(removed).toHaveLength(1);
expect(removed[0]).toContain('.staging-old');
await expect(fs.access(path.join(root, '.staging-old'))).rejects.toBeTruthy();
// Recent staging and the promoted repo dir survive.
await expect(fs.access(path.join(root, '.staging-new'))).resolves.toBeUndefined();
await expect(fs.access(path.join(root, 'myrepo'))).resolves.toBeUndefined();
});
it('tolerates a missing root', async () => {
const { removed } = await sweepStaleUploads({ root: path.join(root, 'does-not-exist') });
expect(removed).toEqual([]);
});
});