diff --git a/gitnexus/src/core/group/extractors/http-patterns/python.ts b/gitnexus/src/core/group/extractors/http-patterns/python.ts index 497840fbb0..70ca21043e 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/python.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/python.ts @@ -504,6 +504,29 @@ function extractFlaskMethods(callNode: Parser.SyntaxNode): string[] { return ['GET']; } +/** + * A consumer URL argument as text. An f/t-string's interpolations become + * `${…}` spans, so the shared consumer normalizer treats a leading one as a + * host binding and later ones as path params, as it does for a JS template + * (#1675, #3111). Other literals go through `unquoteLiteral`. + */ +function pythonUrlLiteral(node: Parser.SyntaxNode): string | null { + const start = node.namedChildren[0]; + if (start?.type !== 'string_start' || !/[fFtT]/.test(start.text)) { + return unquoteLiteral(node.text); + } + let url = ''; + for (const part of node.namedChildren) { + if (part.type === 'string_content') url += part.text.replace(/([{}])\1/g, '$1'); + if (part.type === 'interpolation') { + // Braces inside the expression (`{d['}']}`) would end the span early. + const expression = (part.childForFieldName('expression')?.text ?? '').replace(/[{}]/g, ''); + url += `\${${expression || '_'}}`; + } + } + return url; +} + // Pre-scan: collect local string assignments (uri = "api/v1/endpoint/") function buildLocalStringMap(tree: Parser.Tree): Map { const map = new Map(); @@ -511,7 +534,7 @@ function buildLocalStringMap(tree: Parser.Tree): Map { const varNode = match.captures.var_name; const valNode = match.captures.var_value; if (!varNode || !valNode) continue; - const val = unquoteLiteral(valNode.text); + const val = pythonUrlLiteral(valNode); if (val === null) continue; map.set(varNode.text, val); } @@ -1381,7 +1404,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { const methodNode = match.captures.method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; - const path = unquoteLiteral(pathNode.text); + const path = pythonUrlLiteral(pathNode); if (path === null) continue; out.push({ role: 'consumer', @@ -1399,7 +1422,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { const methodNode = match.captures.method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; - const path = unquoteLiteral(pathNode.text); + const path = pythonUrlLiteral(pathNode); if (path === null) continue; out.push({ role: 'consumer', @@ -1418,7 +1441,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; const methodRaw = unquoteLiteral(methodNode.text); - const path = unquoteLiteral(pathNode.text); + const path = pythonUrlLiteral(pathNode); if (methodRaw === null || path === null) continue; out.push({ role: 'consumer', @@ -1438,7 +1461,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { const pathNode = match.captures.path; if (!clientNode || !methodNode || !pathNode) continue; if (!hasTrackedHttpxAsyncClient(httpxAsyncClients, clientNode)) continue; - const path = unquoteLiteral(pathNode.text); + const path = pythonUrlLiteral(pathNode); if (path === null) continue; out.push({ role: 'consumer', @@ -1459,7 +1482,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { if (!clientNode || !methodNode || !pathNode) continue; if (!hasTrackedHttpxAsyncClient(httpxAsyncClients, clientNode)) continue; const methodRaw = unquoteLiteral(methodNode.text); - const path = unquoteLiteral(pathNode.text); + const path = pythonUrlLiteral(pathNode); if (methodRaw === null || path === null) continue; out.push({ role: 'consumer', @@ -1480,7 +1503,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { const methodNode = match.captures.method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; - const path = unquoteLiteral(pathNode.text); + const path = pythonUrlLiteral(pathNode); if (path === null) continue; // Deduplicate: the two pattern branches can match the same call. Key on @@ -1518,14 +1541,17 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { seenVarDetections.add(dedupKey); const resolved = localStrings.get(pathVarNode.text); if (!resolved) continue; - const normalized = normalizeConsumerPath(resolved); - if (normalized === '/') continue; + // A root url is skipped, including one behind a stripped host binding. + if (normalizeConsumerPath(resolved.replace(/^(?:\$\{[^}]*\})+/, '')) === '/') continue; const httpMethod = WRAPPER_METHOD_TO_HTTP[methodNode.text.toLowerCase()] ?? 'GET'; out.push({ role: 'consumer', framework: 'python-http-wrapper', method: httpMethod, - path: normalized, + // Raw, like the literal sites: a leading `${…}` binding is stripped by + // the shared consumer normalizer, which this local one would turn into + // a `{param}` segment first. + path: resolved, name: null, line: methodNode.startPosition.row + 1, confidence: 0.6, diff --git a/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts b/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts index 0be74e6513..b9497fc65a 100644 --- a/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts +++ b/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts @@ -163,7 +163,9 @@ export function scanFile( /** * Strip enclosing quotes from a tree-sitter string literal node's text. * Handles single / double / template quotes, Python triple-quoted strings, - * and Go raw string literals (backticks). + * Python string prefixes (`r`, `b`, `u`, `f`, `t` and pairs such as `rb`), + * and Go raw string literals (backticks). An f-string's `{…}` fields are + * left as text; plugins that need them as interpolations read the node. * * Returns null for empty/nullish input so callers can uniformly skip * captures whose value is missing. @@ -171,18 +173,23 @@ export function scanFile( export function unquoteLiteral(raw: string): string | null { if (!raw) return null; + // Python string prefixes (#1675) and PHP's binary `b"…"` are the only + // letters before an opening quote; a prefix with no matching quote pair + // after it falls through to the raw return below. + const literal = raw.replace(/^[rRbBuUfFtT]{1,2}(?=["'])/, ''); + // Python triple-quoted if ( - (raw.startsWith('"""') && raw.endsWith('"""')) || - (raw.startsWith("'''") && raw.endsWith("'''")) + (literal.startsWith('"""') && literal.endsWith('"""')) || + (literal.startsWith("'''") && literal.endsWith("'''")) ) { - return raw.slice(3, -3); + return literal.slice(3, -3); } - const first = raw[0]; - const last = raw[raw.length - 1]; - if ((first === '"' || first === "'" || first === '`') && last === first && raw.length >= 2) { - return raw.slice(1, -1); + const first = literal[0]; + const last = literal[literal.length - 1]; + if ((first === '"' || first === "'" || first === '`') && last === first && literal.length >= 2) { + return literal.slice(1, -1); } // Some grammars expose the string content without quotes already (e.g. diff --git a/gitnexus/test/unit/group/http-route-extractor.test.ts b/gitnexus/test/unit/group/http-route-extractor.test.ts index 9a619622cb..2d21914f5c 100644 --- a/gitnexus/test/unit/group/http-route-extractor.test.ts +++ b/gitnexus/test/unit/group/http-route-extractor.test.ts @@ -2946,6 +2946,61 @@ def create_order(): consumers.find((c) => c.contractId === 'http::POST::/api/orders/{param}'), ).toBeDefined(); }); + + it('reads prefixed Python literals and f-string URLs as paths (#1675)', async () => { + const dir = path.join(tmpDir, 'python-prefixed-consumer'); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'client.py'), + ` +import requests + +BASE = "https://svc.local" + +def calls(svc, item_id): + requests.get(f"{BASE}/api/items") + requests.get(F"/api/items/{item_id:d}") + requests.post(rf"/api/raw") + requests.put(b"/api/bytes") + requests.delete(u'/api/legacy') + requests.request("PATCH", f"https://svc.local/api/abs/{item_id!r}") + requests.get(url=f"""{BASE}/api/keyword""") + requests.get(f"{BASE}api/relative") + requests.get(f"/api/odd/{ {} }") + uri = f"{BASE}/api/wrapped/{item_id}" + svc.fetch(uri=uri) + relative = "api/v1/relative" + svc.fetch(uri=relative) + root = f"{BASE}/" + svc.fetch(uri=root) +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const consumerIds = contracts + .filter((c) => c.role === 'consumer') + .map((c) => c.contractId) + .sort(); + + // A leading interpolation followed by `/` is a host binding, as for a JS + // template (#3111); one with no `/` after it is not a routable path, and + // a variable url that is only the host is skipped like a literal root. + expect(consumerIds).toEqual( + [ + 'http::DELETE::/api/legacy', + 'http::GET::/api/items', + 'http::GET::/api/items/{param}', + 'http::GET::/api/keyword', + 'http::GET::/api/odd/{param}', + 'http::GET::/api/v1/relative', + 'http::GET::/api/wrapped/{param}', + 'http::PATCH::/api/abs/{param}', + 'http::POST::/api/raw', + 'http::PUT::/api/bytes', + ].sort(), + ); + }); + it('extracts Python httpx.AsyncClient calls assigned to attributes or aliases', async () => { const dir = path.join(tmpDir, 'python-httpx-consumer'); fs.mkdirSync(path.join(dir, 'src'), { recursive: true });