From 33f18ceaa22529eb5d80fd694a3c67bc1c878bef Mon Sep 17 00:00:00 2001 From: Shane Thurston Wijaya <129602553+sanguine59@users.noreply.github.com> Date: Mon, 18 May 2026 22:00:42 +0700 Subject: [PATCH] feat(eval-server): added --host for user configured host IP instead of system hardcoded IP (127.0.0.1) (#1667) * feat(eval-server): added --host for user configured host IP instead of system hardcoded IP (127.0.0.1) * fix(eval-server): localhost value in --host now returns 127.0.0.1 instead of the raw input to fix wrong address, handled error for ipv6 disabled containers * feat(eval-server): add --host flag with validation and error handling Co-Authored-By: Val Vladescu * fix(eval-server): bracketed IPv6 addresses to remove ambiguity * docs(eval-server): document --host flag, READY signal format, and parser migration note * fix(eval-server): use actual bound port in READY signal; strengthen --host e2e tests Co-Authored-By: Val Vladescu * feat(eval): wire eval-server --host through gitnexus_docker.py * docs(eval): added guidance for docker user * docs(eval): revise the imprecise documentation * fix(e2e): updated original stdout for new format --- eval/README.md | 57 +++++++- eval/environments/gitnexus_docker.py | 23 ++- gitnexus/src/cli/eval-server.ts | 99 +++++++++++-- gitnexus/src/cli/index.ts | 4 + gitnexus/test/integration/cli-e2e.test.ts | 155 ++++++++++++++++++++- gitnexus/test/unit/eval-formatters.test.ts | 48 +++++++ 6 files changed, 369 insertions(+), 17 deletions(-) diff --git a/eval/README.md b/eval/README.md index bd9dc4e20..e3539cf20 100644 --- a/eval/README.md +++ b/eval/README.md @@ -162,8 +162,8 @@ Each mode has a `system_{mode}.jinja` + `instance_{mode}.jinja` pair. The agent ``` Agent → bash command → /usr/local/bin/gitnexus-query - → curl localhost:4848/tool/query (fast path: eval-server, ~100ms) - → npx gitnexus query (fallback: cold CLI, ~5-10s) + → curl http://127.0.0.1:4848/tool/query (fast path: eval-server, ~100ms) + → npx gitnexus query (fallback: cold CLI, ~5-10s) ``` Each tool script in `/usr/local/bin/` is standalone — no sourcing, no env inheritance needed. This is critical because mini-swe-agent runs every command via `subprocess.run` in a fresh subshell. @@ -176,6 +176,59 @@ The eval-server is a lightweight HTTP daemon that: - Includes next-step hints to guide tool chaining (query → context → impact → fix) - Auto-shuts down after idle timeout +**CLI flags:** + +| Flag | Default | Purpose | +|------|---------|---------| +| `--port ` | `4848` | Port to listen on | +| `--host ` | `127.0.0.1` | Bind address — use `0.0.0.0` for cross-container access | +| `--idle-timeout ` | `0` (disabled) | Auto-shutdown after N seconds of inactivity | + +**READY signal:** + +When the server is ready, it writes to stdout: + +``` +# IPv4 +GITNEXUS_EVAL_SERVER_READY:127.0.0.1:4848 + +# IPv6 (bracketed to avoid colon ambiguity) +GITNEXUS_EVAL_SERVER_READY:[::1]:4848 +``` + +Parse the port as the last colon-segment (`split(':').pop()`) — not `split(':')[1]`, which breaks for IPv6 and for non-loopback IPv4 hosts added in this release. + +### Custom port and host + +`run_eval.py` does not expose `--port` or `--host` as CLI flags. Configure them in your mode YAML under the `environment:` key: + +```yaml +# configs/modes/native_augment.yaml (or whichever mode you're running) +environment: + eval_server_port: 4849 # change if 4848 is already in use on the host + eval_server_host: "0.0.0.0" # bind all interfaces — needed for cross-container setups +``` + +Defaults are `port: 4848` and `host: 127.0.0.1` (loopback only). Use `0.0.0.0` only when the agent container needs to reach the eval-server from a separate network namespace. The health probe and tool scripts connect via the configured bind host (defaulting to `127.0.0.1`), which is reachable for both loopback and all-interface binds. + +**Running eval-server directly in Docker / Docker Compose:** + +```bash +# Bind to all interfaces so sibling containers can reach it +gitnexus eval-server --host 0.0.0.0 --port 4848 + +# Then probe from a sibling container via its service hostname +curl http://eval-container:4848/health +``` + +If you need a non-default port (e.g. to avoid conflicts), pass `--port ` alongside `--host`. The READY signal will reflect both: + +``` +GITNEXUS_EVAL_SERVER_READY:0.0.0.0:5000 +``` + +Parse the port as the last colon-segment (`split(':').pop()`) — safe for both IPv4 and bracketed IPv6 forms. + ### Index caching SWE-bench repos repeat (Django has 200+ instances at different commits). The harness caches GitNexus indexes per `(repo, commit)` hash in `~/.gitnexus-eval-cache/` to avoid redundant re-indexing. diff --git a/eval/environments/gitnexus_docker.py b/eval/environments/gitnexus_docker.py index 73aec0dd9..2f9640f96 100644 --- a/eval/environments/gitnexus_docker.py +++ b/eval/environments/gitnexus_docker.py @@ -39,6 +39,7 @@ logger = logging.getLogger("gitnexus_docker") DEFAULT_CACHE_DIR = Path.home() / ".gitnexus-eval-cache" EVAL_SERVER_PORT = 4848 +EVAL_SERVER_HOST = "127.0.0.1" class GitNexusDockerEnvironment(DockerEnvironment): @@ -62,6 +63,7 @@ class GitNexusDockerEnvironment(DockerEnvironment): skip_embeddings: bool = True, gitnexus_timeout: int = 120, eval_server_port: int = EVAL_SERVER_PORT, + eval_server_host: str = EVAL_SERVER_HOST, **kwargs, ): super().__init__(**kwargs) @@ -70,6 +72,7 @@ class GitNexusDockerEnvironment(DockerEnvironment): self.skip_embeddings = skip_embeddings self.gitnexus_timeout = gitnexus_timeout self.eval_server_port = eval_server_port + self.eval_server_host = eval_server_host self.index_time: float = 0.0 self._gitnexus_ready = False @@ -165,22 +168,29 @@ class GitNexusDockerEnvironment(DockerEnvironment): def _start_eval_server(self): """Start the GitNexus eval-server daemon in the background.""" - logger.info(f"Starting eval-server on port {self.eval_server_port}...") + logger.info( + f"Starting eval-server on {self.eval_server_host}:{self.eval_server_port}..." + ) self.execute({ "command": ( f"nohup npx gitnexus eval-server --port {self.eval_server_port} " + f"--host {self.eval_server_host} " f"--idle-timeout 600 " f"> /tmp/gitnexus-eval-server.log 2>&1 &" ), "timeout": 5, }) + # Use 127.0.0.1 for the health probe — reachable whether server binds + # loopback or all interfaces (0.0.0.0), avoiding DNS resolution issues. + health_host = "127.0.0.1" + # Wait for the server to be ready (up to ~15s for KuzuDB init) for i in range(EVAL_SERVER_HEALTH_RETRIES): time.sleep(EVAL_SERVER_HEALTH_INTERVAL_SECONDS) health = self.execute({ - "command": f"curl -sf http://127.0.0.1:{self.eval_server_port}/health 2>/dev/null || echo 'NOT_READY'", + "command": f"curl -sf http://{health_host}:{self.eval_server_port}/health 2>/dev/null || echo 'NOT_READY'", "timeout": EVAL_SERVER_HEALTH_TIMEOUT_SECONDS, }) output = health.get("output", "").strip() @@ -201,7 +211,7 @@ class GitNexusDockerEnvironment(DockerEnvironment): ) @staticmethod - def _render_tool_script(spec: ToolScriptSpec, port: str) -> str: + def _render_tool_script(spec: ToolScriptSpec, port: str, host: str = EVAL_SERVER_HOST) -> str: """ Render a standalone bash script for a GitNexus tool. @@ -212,6 +222,7 @@ class GitNexusDockerEnvironment(DockerEnvironment): if spec.endpoint: lines.append(f'PORT="${{GITNEXUS_EVAL_PORT:-{port}}}"') + lines.append(f'HOST="${{GITNEXUS_EVAL_HOST:-{host}}}"') if spec.header: lines.append(spec.header.strip()) @@ -221,7 +232,7 @@ class GitNexusDockerEnvironment(DockerEnvironment): if spec.endpoint: lines.append( - f'result=$(curl -sf -X POST "http://127.0.0.1:${{PORT}}{spec.endpoint}" ' + f'result=$(curl -sf -X POST "http://${{HOST}}:${{PORT}}{spec.endpoint}" ' '-H "Content-Type: application/json" -d "$payload" 2>/dev/null)' ) lines.append('if [ $? -eq 0 ] && [ -n "$result" ]; then echo "$result"; exit 0; fi') @@ -244,9 +255,10 @@ class GitNexusDockerEnvironment(DockerEnvironment): Uses heredocs with quoted delimiter to avoid all quoting/escaping issues. """ port = str(self.eval_server_port) + host = self.eval_server_host for spec in TOOL_SPECS.values(): - script_content = self._render_tool_script(spec, port).strip() + script_content = self._render_tool_script(spec, port, host).strip() # Use heredoc with quoted delimiter — prevents all variable expansion and quoting issues self.execute({ "command": ( @@ -387,5 +399,6 @@ class GitNexusDockerEnvironment(DockerEnvironment): "index_time_seconds": round(self.index_time, 2), "skip_embeddings": self.skip_embeddings, "eval_server_port": self.eval_server_port, + "eval_server_host": self.eval_server_host, } return base diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index 0819c33e8..8735abbef 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -14,9 +14,14 @@ * Agent bash cmd → curl localhost:PORT/tool/query → eval-server → LocalBackend → format → text * * Usage: - * gitnexus eval-server # default port 4848 - * gitnexus eval-server --port 4848 # explicit port - * gitnexus eval-server --idle-timeout 300 # auto-shutdown after 300s idle + * gitnexus eval-server # default port 4848, binds 127.0.0.1 + * gitnexus eval-server --port 4848 # explicit port + * gitnexus eval-server --host 0.0.0.0 # reachable from other VMs / containers + * gitnexus eval-server --idle-timeout 300 # auto-shutdown after 300s idle + * + * READY signal format: GITNEXUS_EVAL_SERVER_READY:: + * IPv4: GITNEXUS_EVAL_SERVER_READY:127.0.0.1:4848 + * IPv6: GITNEXUS_EVAL_SERVER_READY:[::1]:4848 * * API: * POST /tool/:name — Call a tool. Body is JSON arguments. Returns formatted text. @@ -25,16 +30,28 @@ */ import http from 'http'; +import { isIPv4, isIPv6 } from 'node:net'; import { writeSync } from 'node:fs'; import { LocalBackend } from '../mcp/local/local-backend.js'; import { logger } from '../core/logger.js'; -import { cliInfo, cliWarn } from './cli-message.js'; +import { cliInfo, cliWarn, cliError } from './cli-message.js'; export interface EvalServerOptions { port?: string; + host?: string; idleTimeout?: string; } +/** + * Validate the --host value. Accepts IPv4, IPv6, or "localhost". + * Returns the normalised host string, or null if invalid. + */ +export function validateHost(raw: string): string | null { + if (raw === 'localhost') return '127.0.0.1'; + if (isIPv4(raw) || isIPv6(raw)) return raw; + return null; +} + // ─── Text Formatters ────────────────────────────────────────────────── // Convert structured JSON results into compact, LLM-friendly text. // Design: minimize tokens, maximize actionability. @@ -330,6 +347,22 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise { + server.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EADDRINUSE') { + cliError( + `\nGitNexus eval-server failed to start:\n` + + ` Port ${port} is already in use.\n\n` + + ` Either:\n` + + ` 1. Stop the process already using port ${port}\n` + + ` 2. Use a different port: gitnexus eval-server --port 4849\n`, + { code: err.code, port, host }, + ); + } else if (err.code === 'EADDRNOTAVAIL') { + const isIPv6Host = isIPv6(host); + cliError( + `\nGitNexus eval-server failed to start:\n` + + ` Address ${host} is not available on this machine.\n\n` + + (isIPv6Host + ? ` IPv6 address ${host} is not reachable — IPv6 may be disabled on this system or container.\n` + + ` Docker containers and many CI environments disable IPv6 by default.\n\n` + : ` The --host value must be an IP assigned to a local network interface.\n` + + ` Run \`ip addr\` (Linux) or \`ipconfig\` (Windows) to list available addresses.\n\n`) + + ` Common fixes:\n` + + ` gitnexus eval-server --host 127.0.0.1 (loopback, this machine only)\n` + + ` gitnexus eval-server --host 0.0.0.0 (all interfaces, reachable from other VMs)\n`, + { code: err.code, port, host }, + ); + } else if (err.code === 'EACCES') { + cliError( + `\nGitNexus eval-server failed to start:\n` + + ` Permission denied binding to port ${port}.\n\n` + + ` Ports below 1024 require elevated privileges.\n` + + ` Use a port above 1024: gitnexus eval-server --port 4848\n`, + { code: err.code, port, host }, + ); + } else { + cliError(`\nGitNexus eval-server failed to start:\n ${err.message}\n`, { + code: err.code, + port, + host, + }); + } + process.exit(1); + }); + + server.listen(port, host, () => { // Plain-text banner for the human watching stderr; structured record // for log aggregation (split into two so the user sees a real banner // not `{"level":30,"msg":"...","port":4747,"endpoints":[...]}`). + // Use server.address().port so --port 0 (OS-assigned) emits the real port. + const addr = server.address(); + const boundPort = typeof addr === 'object' && addr !== null ? addr.port : port; + const displayHost = host.includes(':') ? `[${host}]` : host; const bannerLines = [ - `GitNexus eval-server: listening on http://127.0.0.1:${port}`, + `GitNexus eval-server: listening on http://${displayHost}:${boundPort}`, ` POST /tool/query — search execution flows`, ` POST /tool/context — 360-degree symbol view`, ` POST /tool/impact — blast radius analysis`, @@ -443,8 +523,8 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise 0 ? idleTimeoutSec : undefined, endpoints: [ 'POST /tool/query', @@ -457,7 +537,8 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise', 'Port number', '4848') + .option( + '--host ', + 'Bind address (default: 127.0.0.1, use 0.0.0.0 to expose to all interfaces)', + ) .option('--idle-timeout ', 'Auto-shutdown after N seconds idle (0 = disabled)', '0') .action(createLazyAction(() => import('./eval-server.js'), 'evalServerCommand')); diff --git a/gitnexus/test/integration/cli-e2e.test.ts b/gitnexus/test/integration/cli-e2e.test.ts index 99e52c330..b739ad070 100644 --- a/gitnexus/test/integration/cli-e2e.test.ts +++ b/gitnexus/test/integration/cli-e2e.test.ts @@ -1217,7 +1217,7 @@ describe('CLI end-to-end', () => { child.stdout.on('data', (chunk: Buffer) => { stdoutBuffer += chunk.toString(); - if (stdoutBuffer.includes('GITNEXUS_EVAL_SERVER_READY:')) { + if (stdoutBuffer.includes('GITNEXUS_EVAL_SERVER_READY:127.0.0.1:')) { foundOnStdout = true; child.kill('SIGTERM'); } @@ -1255,4 +1255,157 @@ describe('CLI end-to-end', () => { }); }, 35000); }); + + // ─── eval-server --host flag tests ─────────────────────────────────── + // Verifies --host is wired to the actual bind address, not just accepted. + // Original flag registration test by Val Vladescu (PR #1602). + + describe('eval-server --host flag', () => { + it('emits READY signal containing the bound host 127.0.0.1', () => { + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + '--import', + tsxImportUrl, + cliEntry, + 'eval-server', + '--port', + '0', + '--host', + '127.0.0.1', + '--idle-timeout', + '3', + ], + { + cwd: MINI_REPO, + stdio: ['ignore', 'pipe', 'pipe'], + env: cliEnv(), + }, + ); + + let stdoutBuffer = ''; + let stderrBuffer = ''; + let settled = false; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.kill('SIGTERM'); + fn(); + }; + + child.stdout.on('data', (chunk: Buffer) => { + stdoutBuffer += chunk.toString(); + if (stdoutBuffer.includes('GITNEXUS_EVAL_SERVER_READY:')) { + if (stdoutBuffer.includes('GITNEXUS_EVAL_SERVER_READY:127.0.0.1:')) { + settle(resolve); + } else { + settle(() => + reject( + new Error( + `READY signal did not contain expected host 127.0.0.1:\n${stdoutBuffer}`, + ), + ), + ); + } + } + }); + + child.stderr.on('data', (chunk: Buffer) => { + stderrBuffer += chunk.toString(); + if (stderrBuffer.includes('unknown option') || stderrBuffer.includes('error: unknown')) { + settle(() => reject(new Error(`eval-server rejected --host flag:\n${stderrBuffer}`))); + } + }); + + const timer = setTimeout(() => { + settle(() => reject(new Error('eval-server did not emit READY signal within 30s'))); + }, 30000); + }); + }, 35000); + + it('binds to 0.0.0.0 and serves /health on 127.0.0.1 (cross-container use case)', () => { + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + '--import', + tsxImportUrl, + cliEntry, + 'eval-server', + '--port', + '0', + '--host', + '0.0.0.0', + '--idle-timeout', + '3', + ], + { + cwd: MINI_REPO, + stdio: ['ignore', 'pipe', 'pipe'], + env: cliEnv(), + }, + ); + + let stdoutBuffer = ''; + let settled = false; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.kill('SIGTERM'); + fn(); + }; + + child.stdout.on('data', async (chunk: Buffer) => { + stdoutBuffer += chunk.toString(); + const readyLine = stdoutBuffer + .split('\n') + .find((l) => l.startsWith('GITNEXUS_EVAL_SERVER_READY:0.0.0.0:')); + if (!readyLine || settled) return; + + // Parse the actual OS-assigned port from the READY signal + const boundPort = readyLine.split(':').pop()?.trim(); + if (!boundPort || isNaN(Number(boundPort))) { + settle(() => reject(new Error(`Could not parse port from READY signal: ${readyLine}`))); + return; + } + + // A server bound to 0.0.0.0 must be reachable on 127.0.0.1 from the same host + try { + const res = await fetch(`http://127.0.0.1:${boundPort}/health`); + if (res.status === 200) { + settle(resolve); + } else { + settle(() => reject(new Error(`/health returned ${res.status}, expected 200`))); + } + } catch (err) { + settle(() => + reject( + new Error( + `eval-server bound to 0.0.0.0 but /health unreachable on 127.0.0.1:${boundPort}: ${err}`, + ), + ), + ); + } + }); + + child.stderr.on('data', (chunk: Buffer) => { + const text = chunk.toString(); + if (text.includes('unknown option') || text.includes('error: unknown')) { + settle(() => reject(new Error(`eval-server rejected --host flag:\n${text}`))); + } + }); + + const timer = setTimeout(() => { + settle(() => + reject(new Error('eval-server --host 0.0.0.0 did not emit READY signal within 30s')), + ); + }, 30000); + }); + }, 35000); + }); }); diff --git a/gitnexus/test/unit/eval-formatters.test.ts b/gitnexus/test/unit/eval-formatters.test.ts index d29af3c52..6cefbcdca 100644 --- a/gitnexus/test/unit/eval-formatters.test.ts +++ b/gitnexus/test/unit/eval-formatters.test.ts @@ -13,8 +13,56 @@ import { formatDetectChangesResult, formatListReposResult, MAX_BODY_SIZE, + validateHost, } from '../../src/cli/eval-server.js'; +// ─── validateHost ──────────────────────────────────────────────────── + +describe('validateHost', () => { + it('normalizes "localhost" to "127.0.0.1"', () => { + expect(validateHost('localhost')).toBe('127.0.0.1'); + }); + + it('accepts valid IPv4 addresses', () => { + expect(validateHost('127.0.0.1')).toBe('127.0.0.1'); + expect(validateHost('0.0.0.0')).toBe('0.0.0.0'); + expect(validateHost('192.168.1.5')).toBe('192.168.1.5'); + expect(validateHost('10.0.0.1')).toBe('10.0.0.1'); + }); + + it('accepts valid IPv6 addresses', () => { + expect(validateHost('::1')).toBe('::1'); + expect(validateHost('::')).toBe('::'); + expect(validateHost('2001:db8::1')).toBe('2001:db8::1'); + }); + + it('returns null for a non-IP hostname', () => { + expect(validateHost('foo.bar')).toBeNull(); + expect(validateHost('myhost.local')).toBeNull(); + expect(validateHost('example.com')).toBeNull(); + }); + + it('returns null for out-of-range IPv4 octets', () => { + expect(validateHost('999.999.999.999')).toBeNull(); + expect(validateHost('192.168.1.256')).toBeNull(); + }); + + it('returns null for incomplete IPv4 addresses', () => { + expect(validateHost('192.168.1')).toBeNull(); + expect(validateHost('192.168')).toBeNull(); + }); + + it('returns null for an empty string', () => { + expect(validateHost('')).toBeNull(); + }); + + it('returns null for whitespace or padded IPs', () => { + expect(validateHost(' ')).toBeNull(); + expect(validateHost(' 127.0.0.1')).toBeNull(); + expect(validateHost('127.0.0.1 ')).toBeNull(); + }); +}); + // ─── MAX_BODY_SIZE ─────────────────────────────────────────────────── describe('MAX_BODY_SIZE', () => {