import { readFile, writeFile } from 'node:fs/promises' function normalizeBaseUrl(baseUrl) { return String(baseUrl || '').trim().replace(/\/$/, '') || 'https://hypertwist.app' } function uniqueTrimmedStrings(values) { const result = [] const seen = new Set() for (const rawValue of Array.isArray(values) ? values : []) { const value = String(rawValue || '').trim() if (!value || seen.has(value)) { continue } seen.add(value) result.push(value) } return result } function readRoutePath(route) { const path = String(route?.path || '').trim() return path || null } export function buildPublicSearchAssetPlan(routeRegistry, policy = {}) { const sanitizedRoutes = Array.isArray(routeRegistry) ? routeRegistry.filter((route) => readRoutePath(route)) : [] const crawlableRoutes = sanitizedRoutes.filter((route) => route?.crawlable === true) const nonCrawlablePaths = sanitizedRoutes .filter((route) => route?.crawlable !== true) .map((route) => readRoutePath(route)) .filter(Boolean) const explicitDisallowPaths = uniqueTrimmedStrings(policy?.explicitDisallowPaths) const crawlerDisallowPaths = uniqueTrimmedStrings([ ...explicitDisallowPaths, ...nonCrawlablePaths, ]) return { baseUrl: normalizeBaseUrl(policy?.baseUrl), crawlableRoutes, crawlerDisallowPaths, } } export function buildPublicSitemapXml(routeRegistry, policy = {}) { const { baseUrl, crawlableRoutes } = buildPublicSearchAssetPlan(routeRegistry, policy) const urlEntries = crawlableRoutes .map((route) => ` \n ${baseUrl}${route.path === '/' ? '/' : route.path}\n `) .join('\n') return `\n\n${urlEntries}\n\n` } export function buildPublicRobotsTxt(routeRegistry, policy = {}) { const { baseUrl, crawlerDisallowPaths } = buildPublicSearchAssetPlan(routeRegistry, policy) return [ 'User-agent: *', 'Allow: /', ...crawlerDisallowPaths.map((path) => `Disallow: ${path}`), '', `Sitemap: ${baseUrl}/sitemap.xml`, '', ].join('\n') } export async function loadPublicSearchAssetInputs({ registryPath, policyPath }) { const [routeRegistry, policy] = await Promise.all([ readFile(registryPath, 'utf8').then((content) => JSON.parse(content)), readFile(policyPath, 'utf8').then((content) => JSON.parse(content)), ]) return { routeRegistry, policy } } export async function renderPublicSearchAssets({ registryPath, policyPath, sitemapPath, robotsPath, }) { const { routeRegistry, policy } = await loadPublicSearchAssetInputs({ registryPath, policyPath, }) const plan = buildPublicSearchAssetPlan(routeRegistry, policy) const sitemapXml = buildPublicSitemapXml(routeRegistry, policy) const robotsTxt = buildPublicRobotsTxt(routeRegistry, policy) await Promise.all([ writeFile(sitemapPath, sitemapXml, 'utf8'), writeFile(robotsPath, robotsTxt, 'utf8'), ]) return { ...plan, sitemapXml, robotsTxt, } }