diff --git a/.env.example b/.env.example index 8af9dee79..445ae37a7 100644 --- a/.env.example +++ b/.env.example @@ -17,3 +17,9 @@ WEB_HOST_PORT=4173 # Optional read-only mount, exposed to the server as /workspace. # Override with the directory that contains the repos you want to index. WORKSPACE_DIR=./ + +# Azure DevOps Server Integration (passed to the server container) +# Prefer https:// — the PAT rides in an Authorization header, so cleartext +# http:// exposes it on the wire (still supported for internal-only instances). +# AZURE_DEVOPS_URL=https://azuredevops.example.com +# AZURE_DEVOPS_PAT=your-pat-here diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index ee0d92332..4b0548e4f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -276,6 +276,24 @@ jobs: run: node --expose-gc --import tsx bench/cfg/measure.mjs --check working-directory: gitnexus + - name: Emit-persistence throughput / byte-identity guards (#2203) + # Build-free: asserts streamAllCSVsToDisk output is byte-identical + # (order-independent CSV-line fingerprint — the #2203 U2/U3 emit + # optimisations must not change graph content) and that emit wall-time + # stays linear in node+edge count. The LadybugDB COPY half needs a real + # DB, so its timing lives in the runtime PROF_LBUG_LOAD breakdown. + run: node --import tsx bench/emit-persistence/measure.mjs --check + working-directory: gitnexus + + - name: Streaming PDG-emit byte-identity / bounded-RSS guards (#2202) + # Build-free: asserts the streaming PdgEmitSink emits a CSV row SET + # byte-identical to the whole-graph streamAllCSVsToDisk emit, AND that + # the in-memory graph retains zero BasicBlock nodes (the O(chunk) peak-RSS + # bound that unblocks full-kernel-scale repos). Fails on fingerprint drift + # or any resident BasicBlock. + run: node --import tsx bench/emit-persistence/measure-streaming.mjs --check + working-directory: gitnexus + - name: Cross-language pipeline benchmarks (GITNEXUS_BENCH, serial) env: GITNEXUS_BENCH: '1' diff --git a/.gitleaks.toml b/.gitleaks.toml index 769b7eee9..cecf77d5a 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -3,10 +3,17 @@ title = "GitNexus" [extend] useDefault = true -# Fake embedding API keys in unit tests (current probe + historical placeholder). +# Fake credentials in unit tests — none are real secrets: +# - embedding API keys in the http-embedder tests (regexes below) +# - synthetic GitHub PAT fixtures in the git-clone PAT-injection tests +# (e.g. ghp_secret123, ghp_uniqueRawSecret_98765) — allowlisted by path +# so the exception is bounded to that one test file. [allowlist] -description = "fake embedding API keys in http-embedder unit tests" +description = "fake credentials in unit tests (no real secrets)" regexes = [ '''secret-key-12345''', '''test-api-key-redaction-check''', ] +paths = [ + '''gitnexus/test/unit/git-clone\.test\.ts''', +] diff --git a/README.md b/README.md index ab01db810..55e5a03a7 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | | `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | | `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | +| `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | | `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | | `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | diff --git a/gitnexus-web/src/components/AnalyzeOnboarding.tsx b/gitnexus-web/src/components/AnalyzeOnboarding.tsx index 53260e388..b970ee842 100644 --- a/gitnexus-web/src/components/AnalyzeOnboarding.tsx +++ b/gitnexus-web/src/components/AnalyzeOnboarding.tsx @@ -3,7 +3,7 @@ * * The "empty state" card rendered inside DropZone's Crossfade when the server * is connected but zero repos are indexed. Replaces the generic error message - * with a first-class GitHub URL input flow. + * with a first-class repository URL input flow. * * Rendering context: * DropZone (Crossfade, phase="analyze") @@ -15,7 +15,7 @@ * the app to the graph explorer. */ -import { Sparkles, Github } from '@/lib/lucide-icons'; +import { Sparkles, GitBranch } from '@/lib/lucide-icons'; import { RepoAnalyzer } from './RepoAnalyzer'; import { useTranslation } from 'react-i18next'; @@ -46,7 +46,7 @@ export const AnalyzeOnboarding = ({ onComplete }: AnalyzeOnboardingProps) => { {/* Icon */}
- +

diff --git a/gitnexus-web/src/components/RepoAnalyzer.tsx b/gitnexus-web/src/components/RepoAnalyzer.tsx index 613284830..fd08d197d 100644 --- a/gitnexus-web/src/components/RepoAnalyzer.tsx +++ b/gitnexus-web/src/components/RepoAnalyzer.tsx @@ -10,12 +10,14 @@ import { useState, useRef, useEffect, useId } from 'react'; import { Github, Gitlab, + AzureDevops, FolderOpen, Loader2, Check, ArrowRight, AlertCircle, Sparkles, + Key, } from '@/lib/lucide-icons'; import { startAnalyze, @@ -30,10 +32,14 @@ import { useTranslation } from 'react-i18next'; // ── Helpers ────────────────────────────────────────────────────────────────── -type InputMode = 'github' | 'gitlab' | 'local'; +type InputMode = 'github' | 'gitlab' | 'azure' | 'local'; const GITHUB_RE = /^https?:\/\/(www\.)?github\.com\/[^/\s]+\/[^/\s]+/i; const GITLAB_RE = /^https?:\/\/[^/\s]+\/[^/\s]+\/[^/\s]+(\/.*)?$/i; +// One-or-more path segments before `/_git/`, so the legacy single-project +// cloud form (myorg.visualstudio.com/project/_git/repo) is accepted too — +// the backend already supports it (isAzureDevOpsUrl / extractRepoName). +const AZURE_RE = /^https?:\/\/[^/\s]+\/(?:[^/\s]+\/)+_git\/[^/\s]+/i; const IS_WINDOWS = navigator.userAgent.toLowerCase().includes('win'); function isValidGithubUrl(value: string): boolean { @@ -44,6 +50,10 @@ function isValidGitlabUrl(value: string): boolean { return GITLAB_RE.test(value.trim()); } +function isValidAzureUrl(value: string): boolean { + return AZURE_RE.test(value.trim()); +} + // ── Mode tabs ──────────────────────────────────────────────────────────────── function ModeTabs({ mode, onChange }: { mode: InputMode; onChange: (m: InputMode) => void }) { @@ -81,6 +91,19 @@ function ModeTabs({ mode, onChange }: { mode: InputMode; onChange: (m: InputMode {t('repoAnalyzer.gitlabUrl')} +