Merge upstream/main (89602560) into g1/r-language-support-rebased

gitnexus/src/core/group/extractors/http-patterns/python.ts

@@ -504,6 +504,29 @@ function extractFlaskMethods(callNode: Parser.SyntaxNode): string[] {
   return ['GET'];
 }
 
+/**
+ * A consumer URL argument as text. An f/t-string's interpolations become
+ * `${…}` spans, so the shared consumer normalizer treats a leading one as a
+ * host binding and later ones as path params, as it does for a JS template
+ * (#1675, #3111). Other literals go through `unquoteLiteral`.
+ */
+function pythonUrlLiteral(node: Parser.SyntaxNode): string | null {
+  const start = node.namedChildren[0];
+  if (start?.type !== 'string_start' || !/[fFtT]/.test(start.text)) {
+    return unquoteLiteral(node.text);
+  }
+  let url = '';
+  for (const part of node.namedChildren) {
+    if (part.type === 'string_content') url += part.text.replace(/([{}])\1/g, '$1');
+    if (part.type === 'interpolation') {
+      // Braces inside the expression (`{d['}']}`) would end the span early.
+      const expression = (part.childForFieldName('expression')?.text ?? '').replace(/[{}]/g, '');
+      url += `\${${expression || '_'}}`;
+    }
+  }
+  return url;
+}
+
 // Pre-scan: collect local string assignments (uri = "api/v1/endpoint/")
 function buildLocalStringMap(tree: Parser.Tree): Map<string, string> {
   const map = new Map<string, string>();
@@ -511,7 +534,7 @@ function buildLocalStringMap(tree: Parser.Tree): Map<string, string> {
     const varNode = match.captures.var_name;
     const valNode = match.captures.var_value;
     if (!varNode || !valNode) continue;
-    const val = unquoteLiteral(valNode.text);
+    const val = pythonUrlLiteral(valNode);
     if (val === null) continue;
     map.set(varNode.text, val);
   }
@@ -1381,7 +1404,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       const methodNode = match.captures.method;
       const pathNode = match.captures.path;
       if (!methodNode || !pathNode) continue;
-      const path = unquoteLiteral(pathNode.text);
+      const path = pythonUrlLiteral(pathNode);
       if (path === null) continue;
       out.push({
         role: 'consumer',
@@ -1399,7 +1422,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       const methodNode = match.captures.method;
       const pathNode = match.captures.path;
       if (!methodNode || !pathNode) continue;
-      const path = unquoteLiteral(pathNode.text);
+      const path = pythonUrlLiteral(pathNode);
       if (path === null) continue;
       out.push({
         role: 'consumer',
@@ -1418,7 +1441,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       const pathNode = match.captures.path;
       if (!methodNode || !pathNode) continue;
       const methodRaw = unquoteLiteral(methodNode.text);
-      const path = unquoteLiteral(pathNode.text);
+      const path = pythonUrlLiteral(pathNode);
       if (methodRaw === null || path === null) continue;
       out.push({
         role: 'consumer',
@@ -1438,7 +1461,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       const pathNode = match.captures.path;
       if (!clientNode || !methodNode || !pathNode) continue;
       if (!hasTrackedHttpxAsyncClient(httpxAsyncClients, clientNode)) continue;
-      const path = unquoteLiteral(pathNode.text);
+      const path = pythonUrlLiteral(pathNode);
       if (path === null) continue;
       out.push({
         role: 'consumer',
@@ -1459,7 +1482,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       if (!clientNode || !methodNode || !pathNode) continue;
       if (!hasTrackedHttpxAsyncClient(httpxAsyncClients, clientNode)) continue;
       const methodRaw = unquoteLiteral(methodNode.text);
-      const path = unquoteLiteral(pathNode.text);
+      const path = pythonUrlLiteral(pathNode);
       if (methodRaw === null || path === null) continue;
       out.push({
         role: 'consumer',
@@ -1480,7 +1503,7 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       const methodNode = match.captures.method;
       const pathNode = match.captures.path;
       if (!methodNode || !pathNode) continue;
-      const path = unquoteLiteral(pathNode.text);
+      const path = pythonUrlLiteral(pathNode);
       if (path === null) continue;
 
       // Deduplicate: the two pattern branches can match the same call. Key on
@@ -1518,14 +1541,17 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
       seenVarDetections.add(dedupKey);
       const resolved = localStrings.get(pathVarNode.text);
       if (!resolved) continue;
-      const normalized = normalizeConsumerPath(resolved);
-      if (normalized === '/') continue;
+      // A root url is skipped, including one behind a stripped host binding.
+      if (normalizeConsumerPath(resolved.replace(/^(?:\$\{[^}]*\})+/, '')) === '/') continue;
       const httpMethod = WRAPPER_METHOD_TO_HTTP[methodNode.text.toLowerCase()] ?? 'GET';
       out.push({
         role: 'consumer',
         framework: 'python-http-wrapper',
         method: httpMethod,
-        path: normalized,
+        // Raw, like the literal sites: a leading `${…}` binding is stripped by
+        // the shared consumer normalizer, which this local one would turn into
+        // a `{param}` segment first.
+        path: resolved,
         name: null,
         line: methodNode.startPosition.row + 1,
         confidence: 0.6,

gitnexus/src/core/group/extractors/tree-sitter-scanner.ts

@@ -163,7 +163,9 @@ export function scanFile<TMeta>(
 /**
  * Strip enclosing quotes from a tree-sitter string literal node's text.
  * Handles single / double / template quotes, Python triple-quoted strings,
- * and Go raw string literals (backticks).
+ * Python string prefixes (`r`, `b`, `u`, `f`, `t` and pairs such as `rb`),
+ * and Go raw string literals (backticks). An f-string's `{…}` fields are
+ * left as text; plugins that need them as interpolations read the node.
  *
  * Returns null for empty/nullish input so callers can uniformly skip
  * captures whose value is missing.
@@ -171,18 +173,23 @@ export function scanFile<TMeta>(
 export function unquoteLiteral(raw: string): string | null {
   if (!raw) return null;
 
+  // Python string prefixes (#1675) and PHP's binary `b"…"` are the only
+  // letters before an opening quote; a prefix with no matching quote pair
+  // after it falls through to the raw return below.
+  const literal = raw.replace(/^[rRbBuUfFtT]{1,2}(?=["'])/, '');
+
   // Python triple-quoted
   if (
-    (raw.startsWith('"""') && raw.endsWith('"""')) ||
-    (raw.startsWith("'''") && raw.endsWith("'''"))
+    (literal.startsWith('"""') && literal.endsWith('"""')) ||
+    (literal.startsWith("'''") && literal.endsWith("'''"))
   ) {
-    return raw.slice(3, -3);
+    return literal.slice(3, -3);
   }
 
-  const first = raw[0];
-  const last = raw[raw.length - 1];
-  if ((first === '"' || first === "'" || first === '`') && last === first && raw.length >= 2) {
-    return raw.slice(1, -1);
+  const first = literal[0];
+  const last = literal[literal.length - 1];
+  if ((first === '"' || first === "'" || first === '`') && last === first && literal.length >= 2) {
+    return literal.slice(1, -1);
   }
 
   // Some grammars expose the string content without quotes already (e.g.

gitnexus/test/unit/group/http-route-extractor.test.ts

@@ -2946,6 +2946,61 @@ def create_order():
         consumers.find((c) => c.contractId === 'http::POST::/api/orders/{param}'),
       ).toBeDefined();
     });
+
+    it('reads prefixed Python literals and f-string URLs as paths (#1675)', async () => {
+      const dir = path.join(tmpDir, 'python-prefixed-consumer');
+      fs.mkdirSync(path.join(dir, 'src'), { recursive: true });
+      fs.writeFileSync(
+        path.join(dir, 'src', 'client.py'),
+        `
+import requests
+
+BASE = "https://svc.local"
+
+def calls(svc, item_id):
+    requests.get(f"{BASE}/api/items")
+    requests.get(F"/api/items/{item_id:d}")
+    requests.post(rf"/api/raw")
+    requests.put(b"/api/bytes")
+    requests.delete(u'/api/legacy')
+    requests.request("PATCH", f"https://svc.local/api/abs/{item_id!r}")
+    requests.get(url=f"""{BASE}/api/keyword""")
+    requests.get(f"{BASE}api/relative")
+    requests.get(f"/api/odd/{ {} }")
+    uri = f"{BASE}/api/wrapped/{item_id}"
+    svc.fetch(uri=uri)
+    relative = "api/v1/relative"
+    svc.fetch(uri=relative)
+    root = f"{BASE}/"
+    svc.fetch(uri=root)
+`,
+      );
+
+      const contracts = await extractor.extract(null, dir, makeRepo(dir));
+      const consumerIds = contracts
+        .filter((c) => c.role === 'consumer')
+        .map((c) => c.contractId)
+        .sort();
+
+      // A leading interpolation followed by `/` is a host binding, as for a JS
+      // template (#3111); one with no `/` after it is not a routable path, and
+      // a variable url that is only the host is skipped like a literal root.
+      expect(consumerIds).toEqual(
+        [
+          'http::DELETE::/api/legacy',
+          'http::GET::/api/items',
+          'http::GET::/api/items/{param}',
+          'http::GET::/api/keyword',
+          'http::GET::/api/odd/{param}',
+          'http::GET::/api/v1/relative',
+          'http::GET::/api/wrapped/{param}',
+          'http::PATCH::/api/abs/{param}',
+          'http::POST::/api/raw',
+          'http::PUT::/api/bytes',
+        ].sort(),
+      );
+    });
+
     it('extracts Python httpx.AsyncClient calls assigned to attributes or aliases', async () => {
       const dir = path.join(tmpDir, 'python-httpx-consumer');
       fs.mkdirSync(path.join(dir, 'src'), { recursive: true });