106 lines
3 KiB
JavaScript
106 lines
3 KiB
JavaScript
import { readFile, writeFile } from 'node:fs/promises'
|
|
|
|
function normalizeBaseUrl(baseUrl) {
|
|
return String(baseUrl || '').trim().replace(/\/$/, '') || 'https://hypertwist.app'
|
|
}
|
|
|
|
function uniqueTrimmedStrings(values) {
|
|
const result = []
|
|
const seen = new Set()
|
|
|
|
for (const rawValue of Array.isArray(values) ? values : []) {
|
|
const value = String(rawValue || '').trim()
|
|
if (!value || seen.has(value)) {
|
|
continue
|
|
}
|
|
|
|
seen.add(value)
|
|
result.push(value)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
function readRoutePath(route) {
|
|
const path = String(route?.path || '').trim()
|
|
return path || null
|
|
}
|
|
|
|
export function buildPublicSearchAssetPlan(routeRegistry, policy = {}) {
|
|
const sanitizedRoutes = Array.isArray(routeRegistry) ? routeRegistry.filter((route) => readRoutePath(route)) : []
|
|
const crawlableRoutes = sanitizedRoutes.filter((route) => route?.crawlable === true)
|
|
const nonCrawlablePaths = sanitizedRoutes
|
|
.filter((route) => route?.crawlable !== true)
|
|
.map((route) => readRoutePath(route))
|
|
.filter(Boolean)
|
|
|
|
const explicitDisallowPaths = uniqueTrimmedStrings(policy?.explicitDisallowPaths)
|
|
const crawlerDisallowPaths = uniqueTrimmedStrings([
|
|
...explicitDisallowPaths,
|
|
...nonCrawlablePaths,
|
|
])
|
|
|
|
return {
|
|
baseUrl: normalizeBaseUrl(policy?.baseUrl),
|
|
crawlableRoutes,
|
|
crawlerDisallowPaths,
|
|
}
|
|
}
|
|
|
|
export function buildPublicSitemapXml(routeRegistry, policy = {}) {
|
|
const { baseUrl, crawlableRoutes } = buildPublicSearchAssetPlan(routeRegistry, policy)
|
|
const urlEntries = crawlableRoutes
|
|
.map((route) => ` <url>\n <loc>${baseUrl}${route.path === '/' ? '/' : route.path}</loc>\n </url>`)
|
|
.join('\n')
|
|
|
|
return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urlEntries}\n</urlset>\n`
|
|
}
|
|
|
|
export function buildPublicRobotsTxt(routeRegistry, policy = {}) {
|
|
const { baseUrl, crawlerDisallowPaths } = buildPublicSearchAssetPlan(routeRegistry, policy)
|
|
|
|
return [
|
|
'User-agent: *',
|
|
'Allow: /',
|
|
...crawlerDisallowPaths.map((path) => `Disallow: ${path}`),
|
|
'',
|
|
`Sitemap: ${baseUrl}/sitemap.xml`,
|
|
'',
|
|
].join('\n')
|
|
}
|
|
|
|
export async function loadPublicSearchAssetInputs({ registryPath, policyPath }) {
|
|
const [routeRegistry, policy] = await Promise.all([
|
|
readFile(registryPath, 'utf8').then((content) => JSON.parse(content)),
|
|
readFile(policyPath, 'utf8').then((content) => JSON.parse(content)),
|
|
])
|
|
|
|
return { routeRegistry, policy }
|
|
}
|
|
|
|
export async function renderPublicSearchAssets({
|
|
registryPath,
|
|
policyPath,
|
|
sitemapPath,
|
|
robotsPath,
|
|
}) {
|
|
const { routeRegistry, policy } = await loadPublicSearchAssetInputs({
|
|
registryPath,
|
|
policyPath,
|
|
})
|
|
|
|
const plan = buildPublicSearchAssetPlan(routeRegistry, policy)
|
|
const sitemapXml = buildPublicSitemapXml(routeRegistry, policy)
|
|
const robotsTxt = buildPublicRobotsTxt(routeRegistry, policy)
|
|
|
|
await Promise.all([
|
|
writeFile(sitemapPath, sitemapXml, 'utf8'),
|
|
writeFile(robotsPath, robotsTxt, 'utf8'),
|
|
])
|
|
|
|
return {
|
|
...plan,
|
|
sitemapXml,
|
|
robotsTxt,
|
|
}
|
|
}
|