Unify HyperTwist public search asset authority

docs/v6_5_deep_manual_pack/HyperTwist/DEVELOPMENT.md

@@ -157,6 +157,12 @@ Current packaged-proof bridge truth after the same `2026-06-23` continuation:
 - the `2026-06-24` centralized public-route-authority packet then kept the
   same umbrella gate green while moving navigation, footer links, router
   entries, and sitemap generation onto a shared route registry
+- the later `2026-06-30` public search-asset authority follow-up then
+  widened that same route-owned lane again so both `website/public/sitemap.xml`
+  and `website/public/robots.txt` are rendered from one shared public-route
+  registry plus explicit crawler-policy authority instead of leaving
+  `robots.txt` as a hand-maintained drift risk, while the owned website gate
+  now also exercises `scripts/render-public-search-assets-lib.test.mjs`
 - `scripts/run-hypertwist-remote-windows-file-pull.sh` now gives the lane a
   bounded first-party way to pull Windows-side validation artifacts back into
   the HyperTwist repo with explicit remote-to-local path mapping and SHA-256
@@ -219,6 +225,50 @@ Latest same-day follow-up later on `2026-06-24`:
 - `scripts/run-hypertwist-gitnexus-status.sh` then again reported the bounded
   mirror `Status: up-to-date`
 
+Latest same-family website hardening follow-up on `2026-06-30`:
+
+- public search assets now come from one shared authority:
+  - `website/src/public-route-registry.json`
+  - `website/src/public-search-policy.json`
+  - `website/scripts/render-public-search-assets-lib.mjs`
+  - `website/scripts/render-public-search-assets.mjs`
+- the generated crawler posture now keeps:
+  - crawlable public routes in `sitemap.xml`
+  - `/app`, `/api/`, `/auth`, `/health`, `/login`, and `/register` in
+    `robots.txt` disallow posture
+- focused coverage stayed green under:
+  - `npm --prefix website test -- --run src/__tests__/public-route-registry.test.ts scripts/render-public-search-assets-lib.test.mjs`
+  - `2` test files passed
+  - `11` tests passed
+- the full current web-surface umbrella stayed green again under:
+  - `scripts/run-hypertwist-web-surface-validation.sh`
+  - focused website route/auth/release validation: `14` test files passed,
+    `84` tests passed
+  - website deployment/readiness tooling validation: `4` test files passed,
+    `34` tests passed
+  - `npm --prefix website run build`
+  - `npm --prefix website/server run type-check`
+  - `npm --prefix website/server test -- --run`
+  - `10` website/server test files passed, `36` tests passed
+  - `npm --prefix Content/Browser run verify:shell`
+  - `npm --prefix Content/Browser run build`
+  - website and `Content/Browser` production audits stayed at
+    `found 0 vulnerabilities`
+  - the documented upstream auth-server residual
+    `supertokens-node -> nodemailer` remained accepted in default mode
+- `scripts/run-hypertwist-sentrux-source-only.sh` then improved again to:
+  - `Quality: 6247`
+  - all `7` rules passing
+- `scripts/run-hypertwist-gitnexus-analyze.sh` then re-indexed the bounded
+  source-only mirror successfully at:
+  - `16,575` nodes
+  - `39,165` edges
+  - `683` clusters
+  - `300` flows
+  - fallback completion time `87.1s`
+- `scripts/run-hypertwist-gitnexus-status.sh` then again reported the bounded
+  mirror `Status: up-to-date`
+
 Latest later same-lane follow-up still on `2026-06-24`:
 
 - the next bounded Unreal refactor packet moved the later

scripts/run-hypertwist-web-surface-validation.sh

@@ -166,6 +166,7 @@ run_step \
 run_step \
   "website deployment/readiness tooling validation" \
   npm --prefix website test -- --run \
+  scripts/render-public-search-assets-lib.test.mjs \
   scripts/runtime-readiness-lib.test.mjs \
   scripts/runtime-readiness-cli.test.mjs \
   scripts/render-same-origin-bundle-lib.test.mjs

website/package.json

@@ -6,7 +6,7 @@
   "type": "module",
   "scripts": {
     "dev": "vite",
-    "build": "npm run render:sitemap && tsc && vite build",
+    "build": "npm run render:public-search-assets && tsc && vite build",
     "preview": "vite preview",
     "type-check": "tsc --noEmit",
     "test": "vitest run",
@@ -14,6 +14,7 @@
     "test:e2e:responsive:list": "playwright test tests/e2e/responsive-public-pages.spec.ts --list",
     "test:e2e:protected-responsive": "playwright test tests/e2e/responsive-protected-app-routes.spec.ts --reporter=line",
     "test:e2e:protected-responsive:list": "playwright test tests/e2e/responsive-protected-app-routes.spec.ts --list",
+    "render:public-search-assets": "node scripts/render-public-search-assets.mjs",
     "render:sitemap": "node scripts/render-public-sitemap.mjs",
     "check:runtime-readiness": "node scripts/check-runtime-readiness.mjs",
     "check:runtime-readiness:preview-live": "node scripts/check-runtime-readiness.mjs --frontend-env .env.preview.example --server-env server/.env.preview.example --health-url https://hypertwist.app --deployment-tier preview",

website/public/robots.txt

@@ -1,6 +1,9 @@
 User-agent: *
 Allow: /
 Disallow: /app
+Disallow: /api/
+Disallow: /auth
+Disallow: /health
 Disallow: /login
 Disallow: /register
 

website/scripts/render-public-search-assets-lib.mjs

@@ -0,0 +1,106 @@
+import { readFile, writeFile } from 'node:fs/promises'
+
+function normalizeBaseUrl(baseUrl) {
+  return String(baseUrl || '').trim().replace(/\/$/, '') || 'https://hypertwist.app'
+}
+
+function uniqueTrimmedStrings(values) {
+  const result = []
+  const seen = new Set()
+
+  for (const rawValue of Array.isArray(values) ? values : []) {
+    const value = String(rawValue || '').trim()
+    if (!value || seen.has(value)) {
+      continue
+    }
+
+    seen.add(value)
+    result.push(value)
+  }
+
+  return result
+}
+
+function readRoutePath(route) {
+  const path = String(route?.path || '').trim()
+  return path || null
+}
+
+export function buildPublicSearchAssetPlan(routeRegistry, policy = {}) {
+  const sanitizedRoutes = Array.isArray(routeRegistry) ? routeRegistry.filter((route) => readRoutePath(route)) : []
+  const crawlableRoutes = sanitizedRoutes.filter((route) => route?.crawlable === true)
+  const nonCrawlablePaths = sanitizedRoutes
+    .filter((route) => route?.crawlable !== true)
+    .map((route) => readRoutePath(route))
+    .filter(Boolean)
+
+  const explicitDisallowPaths = uniqueTrimmedStrings(policy?.explicitDisallowPaths)
+  const crawlerDisallowPaths = uniqueTrimmedStrings([
+    ...explicitDisallowPaths,
+    ...nonCrawlablePaths,
+  ])
+
+  return {
+    baseUrl: normalizeBaseUrl(policy?.baseUrl),
+    crawlableRoutes,
+    crawlerDisallowPaths,
+  }
+}
+
+export function buildPublicSitemapXml(routeRegistry, policy = {}) {
+  const { baseUrl, crawlableRoutes } = buildPublicSearchAssetPlan(routeRegistry, policy)
+  const urlEntries = crawlableRoutes
+    .map((route) => `  <url>\n    <loc>${baseUrl}${route.path === '/' ? '/' : route.path}</loc>\n  </url>`)
+    .join('\n')
+
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urlEntries}\n</urlset>\n`
+}
+
+export function buildPublicRobotsTxt(routeRegistry, policy = {}) {
+  const { baseUrl, crawlerDisallowPaths } = buildPublicSearchAssetPlan(routeRegistry, policy)
+
+  return [
+    'User-agent: *',
+    'Allow: /',
+    ...crawlerDisallowPaths.map((path) => `Disallow: ${path}`),
+    '',
+    `Sitemap: ${baseUrl}/sitemap.xml`,
+    '',
+  ].join('\n')
+}
+
+export async function loadPublicSearchAssetInputs({ registryPath, policyPath }) {
+  const [routeRegistry, policy] = await Promise.all([
+    readFile(registryPath, 'utf8').then((content) => JSON.parse(content)),
+    readFile(policyPath, 'utf8').then((content) => JSON.parse(content)),
+  ])
+
+  return { routeRegistry, policy }
+}
+
+export async function renderPublicSearchAssets({
+  registryPath,
+  policyPath,
+  sitemapPath,
+  robotsPath,
+}) {
+  const { routeRegistry, policy } = await loadPublicSearchAssetInputs({
+    registryPath,
+    policyPath,
+  })
+
+  const plan = buildPublicSearchAssetPlan(routeRegistry, policy)
+  const sitemapXml = buildPublicSitemapXml(routeRegistry, policy)
+  const robotsTxt = buildPublicRobotsTxt(routeRegistry, policy)
+
+  await Promise.all([
+    writeFile(sitemapPath, sitemapXml, 'utf8'),
+    writeFile(robotsPath, robotsTxt, 'utf8'),
+  ])
+
+  return {
+    ...plan,
+    sitemapXml,
+    robotsTxt,
+  }
+}

website/scripts/render-public-search-assets-lib.test.mjs

@@ -0,0 +1,104 @@
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+
+import {
+  buildPublicRobotsTxt,
+  buildPublicSearchAssetPlan,
+  buildPublicSitemapXml,
+  renderPublicSearchAssets,
+} from './render-public-search-assets-lib.mjs'
+
+function createRouteRegistry() {
+  return [
+    { path: '/', crawlable: true },
+    { path: '/features', crawlable: true },
+    { path: '/login', crawlable: false },
+    { path: '/register', crawlable: false },
+  ]
+}
+
+function createPolicy() {
+  return {
+    baseUrl: 'https://hypertwist.app/',
+    explicitDisallowPaths: ['/app', '/api/', '/auth', '/health', '/app'],
+  }
+}
+
+describe('buildPublicSearchAssetPlan', () => {
+  it('derives crawlable routes and crawler disallow rules from shared inputs', () => {
+    const plan = buildPublicSearchAssetPlan(createRouteRegistry(), createPolicy())
+
+    expect(plan.baseUrl).toBe('https://hypertwist.app')
+    expect(plan.crawlableRoutes.map((route) => route.path)).toEqual(['/', '/features'])
+    expect(plan.crawlerDisallowPaths).toEqual([
+      '/app',
+      '/api/',
+      '/auth',
+      '/health',
+      '/login',
+      '/register',
+    ])
+  })
+})
+
+describe('buildPublicSitemapXml', () => {
+  it('renders crawlable routes into sitemap XML', () => {
+    const xml = buildPublicSitemapXml(createRouteRegistry(), createPolicy())
+
+    expect(xml).toContain('<loc>https://hypertwist.app/</loc>')
+    expect(xml).toContain('<loc>https://hypertwist.app/features</loc>')
+    expect(xml).not.toContain('/login')
+    expect(xml).not.toContain('/register')
+  })
+})
+
+describe('buildPublicRobotsTxt', () => {
+  it('renders the shared protected-route boundary into robots posture', () => {
+    const robots = buildPublicRobotsTxt(createRouteRegistry(), createPolicy())
+
+    expect(robots).toContain('User-agent: *')
+    expect(robots).toContain('Allow: /')
+    expect(robots).toContain('Disallow: /app')
+    expect(robots).toContain('Disallow: /api/')
+    expect(robots).toContain('Disallow: /auth')
+    expect(robots).toContain('Disallow: /health')
+    expect(robots).toContain('Disallow: /login')
+    expect(robots).toContain('Disallow: /register')
+    expect(robots).toContain('Sitemap: https://hypertwist.app/sitemap.xml')
+  })
+})
+
+describe('renderPublicSearchAssets', () => {
+  it('writes both sitemap and robots assets from the same authority inputs', async () => {
+    const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hypertwist-public-search-assets-'))
+    const registryPath = path.join(tempRoot, 'public-route-registry.json')
+    const policyPath = path.join(tempRoot, 'public-search-policy.json')
+    const sitemapPath = path.join(tempRoot, 'sitemap.xml')
+    const robotsPath = path.join(tempRoot, 'robots.txt')
+
+    fs.writeFileSync(registryPath, JSON.stringify(createRouteRegistry(), null, 2))
+    fs.writeFileSync(policyPath, JSON.stringify(createPolicy(), null, 2))
+
+    const rendered = await renderPublicSearchAssets({
+      registryPath,
+      policyPath,
+      sitemapPath,
+      robotsPath,
+    })
+
+    expect(rendered.crawlableRoutes).toHaveLength(2)
+    expect(rendered.crawlerDisallowPaths).toEqual([
+      '/app',
+      '/api/',
+      '/auth',
+      '/health',
+      '/login',
+      '/register',
+    ])
+    expect(fs.readFileSync(sitemapPath, 'utf8')).toContain('<loc>https://hypertwist.app/features</loc>')
+    expect(fs.readFileSync(robotsPath, 'utf8')).toContain('Disallow: /health')
+  })
+})

website/scripts/render-public-search-assets.mjs

@@ -0,0 +1,22 @@
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { renderPublicSearchAssets } from './render-public-search-assets-lib.mjs'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+const registryPath = path.join(__dirname, '..', 'src', 'public-route-registry.json')
+const policyPath = path.join(__dirname, '..', 'src', 'public-search-policy.json')
+const sitemapPath = path.join(__dirname, '..', 'public', 'sitemap.xml')
+const robotsPath = path.join(__dirname, '..', 'public', 'robots.txt')
+
+const rendered = await renderPublicSearchAssets({
+  registryPath,
+  policyPath,
+  sitemapPath,
+  robotsPath,
+})
+
+console.log(
+  `Rendered public search assets for ${rendered.crawlableRoutes.length} crawlable routes and ${rendered.crawlerDisallowPaths.length} crawler disallow rules.`,
+)

website/scripts/render-public-sitemap.mjs

@@ -1,24 +1 @@
-import { readFile, writeFile } from 'node:fs/promises'
-import path from 'node:path'
-import { fileURLToPath } from 'node:url'
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url))
-const registryPath = path.join(__dirname, '..', 'src', 'public-route-registry.json')
-const sitemapPath = path.join(__dirname, '..', 'public', 'sitemap.xml')
-const baseUrl = 'https://hypertwist.app'
-
-const routeRegistry = JSON.parse(await readFile(registryPath, 'utf8'))
-const crawlableRoutes = routeRegistry.filter((route) => route && route.crawlable === true)
-
-const xml = [
-  '<?xml version="1.0" encoding="UTF-8"?>',
-  '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
-  ...crawlableRoutes.map((route) => (
-    `  <url>\n    <loc>${baseUrl}${route.path === '/' ? '/' : route.path}</loc>\n  </url>`
-  )),
-  '</urlset>',
-  '',
-].join('\n')
-
-await writeFile(sitemapPath, xml, 'utf8')
-console.log(`Rendered sitemap for ${crawlableRoutes.length} crawlable routes to ${sitemapPath}`)
+import './render-public-search-assets.mjs'

website/src/__tests__/public-route-registry.test.ts

@@ -1,9 +1,11 @@
 import { describe, expect, it } from 'vitest'
 import {
+  buildPublicRobotsTxt,
   buildPublicSitemapXml,
   crawlablePublicRoutes,
   footerLinks,
   marketingNavLinks,
+  publicCrawlerDisallowPaths,
   publicRouteRegistry,
 } from '../public-route-registry'
 
@@ -37,6 +39,17 @@ describe('public route registry', () => {
     expect(crawlablePublicRoutes.map((route) => route.path)).toContain('/launch-status')
   })
 
+  it('keeps protected runtime and auth boundaries out of crawler posture', () => {
+    expect(publicCrawlerDisallowPaths).toEqual([
+      '/app',
+      '/api/',
+      '/auth',
+      '/health',
+      '/login',
+      '/register',
+    ])
+  })
+
   it('carries non-empty loader metadata for every shared public-shell route', () => {
     for (const route of publicRouteRegistry) {
       expect(route.loaderTitle.trim().length).toBeGreaterThan(0)
@@ -55,4 +68,19 @@ describe('public route registry', () => {
     expect(xml).not.toContain('/login')
     expect(xml).not.toContain('/register')
   })
+
+  it('renders robots.txt from the shared crawler policy and route registry', () => {
+    const robots = buildPublicRobotsTxt()
+
+    expect(robots).toContain('User-agent: *')
+    expect(robots).toContain('Allow: /')
+    expect(robots).toContain('Disallow: /app')
+    expect(robots).toContain('Disallow: /api/')
+    expect(robots).toContain('Disallow: /auth')
+    expect(robots).toContain('Disallow: /health')
+    expect(robots).toContain('Disallow: /login')
+    expect(robots).toContain('Disallow: /register')
+    expect(robots).toContain('Sitemap: https://hypertwist.app/sitemap.xml')
+    expect(robots).not.toContain('Disallow: /features')
+  })
 })

website/src/public-route-registry.ts

@@ -1,4 +1,5 @@
 import routeRegistry from './public-route-registry.json'
+import publicSearchPolicy from './public-search-policy.json'
 
 export type PublicRouteDefinition = {
   path: string
@@ -11,7 +12,30 @@ export type PublicRouteDefinition = {
   crawlable: boolean
 }
 
+type PublicSearchPolicyDefinition = {
+  baseUrl: string
+  explicitDisallowPaths: string[]
+}
+
 export const publicRouteRegistry = routeRegistry as PublicRouteDefinition[]
+export const searchCrawlerPolicy = publicSearchPolicy as PublicSearchPolicyDefinition
+
+function uniqueTrimmedPaths(paths: readonly string[]) {
+  const result: string[] = []
+  const seen = new Set<string>()
+
+  for (const rawPath of paths) {
+    const path = String(rawPath || '').trim()
+    if (!path || seen.has(path)) {
+      continue
+    }
+
+    seen.add(path)
+    result.push(path)
+  }
+
+  return result
+}
 
 export const marketingNavLinks = publicRouteRegistry
   .filter((route) => route.nav)
@@ -28,8 +52,14 @@ export const footerLinks = publicRouteRegistry
   }))
 
 export const crawlablePublicRoutes = publicRouteRegistry.filter((route) => route.crawlable)
+export const nonCrawlablePublicRoutes = publicRouteRegistry.filter((route) => !route.crawlable)
+export const publicSearchBaseUrl = String(searchCrawlerPolicy.baseUrl || '').trim().replace(/\/$/, '') || 'https://hypertwist.app'
+export const publicCrawlerDisallowPaths = uniqueTrimmedPaths([
+  ...(searchCrawlerPolicy.explicitDisallowPaths || []),
+  ...nonCrawlablePublicRoutes.map((route) => route.path),
+])
 
-export function buildPublicSitemapXml(baseUrl = 'https://hypertwist.app') {
+export function buildPublicSitemapXml(baseUrl = publicSearchBaseUrl) {
   const normalizedBaseUrl = baseUrl.replace(/\/$/, '')
   const urlEntries = crawlablePublicRoutes
     .map((route) => `  <url>\n    <loc>${normalizedBaseUrl}${route.path === '/' ? '/' : route.path}</loc>\n  </url>`)
@@ -37,3 +67,16 @@ export function buildPublicSitemapXml(baseUrl = 'https://hypertwist.app') {
 
   return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urlEntries}\n</urlset>\n`
 }
+
+export function buildPublicRobotsTxt(baseUrl = publicSearchBaseUrl) {
+  const normalizedBaseUrl = baseUrl.replace(/\/$/, '')
+
+  return [
+    'User-agent: *',
+    'Allow: /',
+    ...publicCrawlerDisallowPaths.map((path) => `Disallow: ${path}`),
+    '',
+    `Sitemap: ${normalizedBaseUrl}/sitemap.xml`,
+    '',
+  ].join('\n')
+}

website/src/public-search-policy.json

@@ -0,0 +1,9 @@
+{
+  "baseUrl": "https://hypertwist.app",
+  "explicitDisallowPaths": [
+    "/app",
+    "/api/",
+    "/auth",
+    "/health"
+  ]
+}