fix(web): stop extractUrls treating email addresses as URLs (#1187)

apps/web/lib/extract-urls.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it } from "bun:test"
+import { extractUrls } from "./url-helpers"
+
+describe("extractUrls", () => {
+	it("extracts bare, markdown, and angle-bracket links", () => {
+		const { urls } = extractUrls(
+			"see https://a.example/one, [docs](https://b.example/two) and <https://c.example/three>",
+		)
+		expect(urls).toEqual([
+			"https://a.example/one",
+			"https://b.example/two",
+			"https://c.example/three",
+		])
+	})
+
+	it("normalizes scheme-less URLs", () => {
+		const { urls } = extractUrls("check supermemory.ai for details")
+		expect(urls).toEqual(["https://supermemory.ai"])
+	})
+
+	it("does not extract URLs from email addresses", () => {
+		const result = extractUrls("email me at john.doe@example.com")
+		expect(result.urls).toEqual([])
+		expect(result.duplicates).toBe(0)
+	})
+
+	it("keeps real URLs while skipping emails in the same text", () => {
+		const { urls } = extractUrls(
+			"email john.doe@example.com or visit https://supermemory.ai",
+		)
+		expect(urls).toEqual(["https://supermemory.ai"])
+	})
+
+	it("skips multiple email addresses", () => {
+		const { urls } = extractUrls(
+			"contacts: a.person@foo.example, b.person@bar.example",
+		)
+		expect(urls).toEqual([])
+	})
+
+	it("strips trailing punctuation", () => {
+		const { urls } = extractUrls("read https://example.com/post.")
+		expect(urls).toEqual(["https://example.com/post"])
+	})
+
+	it("dedupes URLs that differ only by scheme/host case or trailing slash", () => {
+		const { urls, duplicates } = extractUrls(
+			"HTTPS://EXAMPLE.COM/docs https://example.com/docs https://example.com/docs/",
+		)
+		expect(urls).toHaveLength(1)
+		expect(duplicates).toBe(2)
+	})
+
+	it("keeps URLs whose paths differ only by case", () => {
+		const { urls, duplicates } = extractUrls(
+			"https://example.com/Page and https://example.com/page",
+		)
+		expect(urls).toEqual([
+			"https://example.com/Page",
+			"https://example.com/page",
+		])
+		expect(duplicates).toBe(0)
+	})
+
+	it("returns nothing for plain text", () => {
+		expect(extractUrls("no links here").urls).toEqual([])
+		expect(extractUrls("").urls).toEqual([])
+	})
+})

apps/web/lib/url-helpers.ts

@@ -96,12 +96,20 @@ export const extractUrls = (
 	const unwrapped = text
 		.replace(MARKDOWN_LINK_REGEX, " $1 ")
 		.replace(ANGLE_LINK_REGEX, " $1 ")
-	const matches = unwrapped.match(URL_TOKEN_REGEX) ?? []
 	const seen = new Set<string>()
 	const urls: string[] = []
 	let duplicates = 0
-	for (const match of matches) {
-		let trimmed = match.trim().replace(/[.,;!]+$/, "")
+	for (const match of unwrapped.matchAll(URL_TOKEN_REGEX)) {
+		const start = match.index ?? 0
+		const end = start + match[0].length
+		const before = start > 0 ? (unwrapped[start - 1] ?? "") : ""
+		const after = end < unwrapped.length ? (unwrapped[end] ?? "") : ""
+		// Skip email addresses: a domain-shaped token ending at "@" is the
+		// local part, one starting right after "@" is the mail domain. Also
+		// skip matches that begin mid-token (e.g. after "_", which the
+		// hostname charset can't include) — those aren't standalone URLs.
+		if (after === "@" || before === "@" || /[\w.-]/.test(before)) continue
+		let trimmed = match[0].trim().replace(/[.,;!]+$/, "")
 		const opens = (trimmed.match(/\(/g) ?? []).length
 		const closes = (trimmed.match(/\)/g) ?? []).length
 		if (closes > opens && trimmed.endsWith(")")) {
@@ -109,7 +117,15 @@ export const extractUrls = (
 		}
 		const normalized = normalizeUrl(trimmed)
 		if (!isValidUrl(normalized)) continue
-		const key = normalized.toLowerCase().replace(/\/+$/, "")
+		// Dedupe on the parsed URL so the scheme and host compare
+		// case-insensitively while the path/query — which are case-sensitive
+		// resources — stay distinct.
+		const parsed = new URL(normalized)
+		const key =
+			`${parsed.origin}${parsed.pathname}${parsed.search}${parsed.hash}`.replace(
+				/\/+$/,
+				"",
+			)
 		if (seen.has(key)) {
 			duplicates++
 			continue