From da38018c17183d42f6aa7a86cf08dfc7301bf872 Mon Sep 17 00:00:00 2001 From: axiomlogicnexus Date: Tue, 30 Jun 2026 05:27:38 +0000 Subject: [PATCH] Unify HyperTwist public search asset authority --- .../HyperTwist/DEVELOPMENT.md | 50 +++++++++ .../HyperTwist/FEATURE_REGISTRY.md | 2 +- .../run-hypertwist-web-surface-validation.sh | 1 + website/package.json | 3 +- website/public/robots.txt | 3 + .../render-public-search-assets-lib.mjs | 106 ++++++++++++++++++ .../render-public-search-assets-lib.test.mjs | 104 +++++++++++++++++ .../scripts/render-public-search-assets.mjs | 22 ++++ website/scripts/render-public-sitemap.mjs | 25 +---- .../__tests__/public-route-registry.test.ts | 28 +++++ website/src/public-route-registry.ts | 45 +++++++- website/src/public-search-policy.json | 9 ++ 12 files changed, 371 insertions(+), 27 deletions(-) create mode 100644 website/scripts/render-public-search-assets-lib.mjs create mode 100644 website/scripts/render-public-search-assets-lib.test.mjs create mode 100644 website/scripts/render-public-search-assets.mjs create mode 100644 website/src/public-search-policy.json diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/DEVELOPMENT.md b/docs/v6_5_deep_manual_pack/HyperTwist/DEVELOPMENT.md index 5f8d797..284b3ea 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/DEVELOPMENT.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/DEVELOPMENT.md @@ -157,6 +157,12 @@ Current packaged-proof bridge truth after the same `2026-06-23` continuation: - the `2026-06-24` centralized public-route-authority packet then kept the same umbrella gate green while moving navigation, footer links, router entries, and sitemap generation onto a shared route registry +- the later `2026-06-30` public search-asset authority follow-up then + widened that same route-owned lane again so both `website/public/sitemap.xml` + and `website/public/robots.txt` are rendered from one shared public-route + registry plus explicit crawler-policy authority instead of leaving + `robots.txt` as a hand-maintained drift risk, while the owned website gate + now also exercises `scripts/render-public-search-assets-lib.test.mjs` - `scripts/run-hypertwist-remote-windows-file-pull.sh` now gives the lane a bounded first-party way to pull Windows-side validation artifacts back into the HyperTwist repo with explicit remote-to-local path mapping and SHA-256 @@ -219,6 +225,50 @@ Latest same-day follow-up later on `2026-06-24`: - `scripts/run-hypertwist-gitnexus-status.sh` then again reported the bounded mirror `Status: up-to-date` +Latest same-family website hardening follow-up on `2026-06-30`: + +- public search assets now come from one shared authority: + - `website/src/public-route-registry.json` + - `website/src/public-search-policy.json` + - `website/scripts/render-public-search-assets-lib.mjs` + - `website/scripts/render-public-search-assets.mjs` +- the generated crawler posture now keeps: + - crawlable public routes in `sitemap.xml` + - `/app`, `/api/`, `/auth`, `/health`, `/login`, and `/register` in + `robots.txt` disallow posture +- focused coverage stayed green under: + - `npm --prefix website test -- --run src/__tests__/public-route-registry.test.ts scripts/render-public-search-assets-lib.test.mjs` + - `2` test files passed + - `11` tests passed +- the full current web-surface umbrella stayed green again under: + - `scripts/run-hypertwist-web-surface-validation.sh` + - focused website route/auth/release validation: `14` test files passed, + `84` tests passed + - website deployment/readiness tooling validation: `4` test files passed, + `34` tests passed + - `npm --prefix website run build` + - `npm --prefix website/server run type-check` + - `npm --prefix website/server test -- --run` + - `10` website/server test files passed, `36` tests passed + - `npm --prefix Content/Browser run verify:shell` + - `npm --prefix Content/Browser run build` + - website and `Content/Browser` production audits stayed at + `found 0 vulnerabilities` + - the documented upstream auth-server residual + `supertokens-node -> nodemailer` remained accepted in default mode +- `scripts/run-hypertwist-sentrux-source-only.sh` then improved again to: + - `Quality: 6247` + - all `7` rules passing +- `scripts/run-hypertwist-gitnexus-analyze.sh` then re-indexed the bounded + source-only mirror successfully at: + - `16,575` nodes + - `39,165` edges + - `683` clusters + - `300` flows + - fallback completion time `87.1s` +- `scripts/run-hypertwist-gitnexus-status.sh` then again reported the bounded + mirror `Status: up-to-date` + Latest later same-lane follow-up still on `2026-06-24`: - the next bounded Unreal refactor packet moved the later diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md index da1f067..7901742 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md @@ -268,7 +268,7 @@ repo. | Feature | Status | Primary authority | Notes | |---|---|---|---| -| Public `hypertwist.app` marketing shell | Implemented now | first-party `website/` app + feature registry/roadmap authority | HyperTwist now has a dedicated first-party public web surface for homepage, feature atlas, about, resources, pricing, download, support, legal routes, a dedicated `/getting-started` onboarding surface, and a dedicated `/launch-status` authority surface. This lane is separate from the embedded Unreal browser runtime under `Content/Browser/` and does not claim browser-simulator parity. The same public lane now also serves as a bounded operator/distribution manual through the docs/resources/download/support surfaces, the public feature-atlas route, the dedicated getting-started route, the dedicated launch-status route, and related launch/status surfaces, explaining browser-versus-desktop posture, rollout steps, package proof, simulator-use guidance, and current input/device truth without claiming browser ownership of the native runtime or overclaiming unfinished VR/controller posture. The same package now also carries a first-party external runtime-readiness verifier so deploy-time env and live health posture can be checked outside the dashboard, plus separated local-versus-production env templates whose placeholder values are intentionally rejected until real launch config is in place, bootstrap CI now validates both the frontend and auth-server website commands directly, and the auth server can now auto-serve the built `website/dist` bundle with bounded SPA fallback for same-origin public deployment. Request-level server coverage now also proves that public/app shell delivery does not shadow `/api/*`, `/auth*`, `/health`, or missing asset paths, while the pricing/download/notices routes now surface first-party preview-versus-launch posture from the same bounded launch checklist instead of relying on hidden operator-only status. The shared marketing shell now also carries a compact public-site-status banner across public pages, the homepage keeps a fuller status section, and the shared public launch-status component now consumes live auth-health webhook/runtime truth in addition to release-manifest download readiness so public marketing copy does not claim launch posture from static checkout/download config alone. The auth server now also emits an authoritative `launch` summary on `/api/auth/health`, including required blocker labels, billing product/price-map gates, and current checkout targets, so the public banner and protected dashboard no longer duplicate launch-blocker evaluation client-side. A later same-family continuation then widened that release-manifest authority again to carry public runtime commerce config for operator/studio checkout URLs and live plan-price strings, so pricing, notices, and dashboard launch-readiness surfaces no longer depend only on frontend build-time checkout config. The live website lane now also owns route-aware title/description/canonical/Open-Graph/Twitter metadata for the real `hypertwist.app` marketing surface so deployed public pages no longer remain on a single generic SPA title/description, plus first-party `robots.txt` and `sitemap.xml` assets for the public route set while keeping `/app`, `/login`, and `/register` out of crawler posture. The real `check-runtime-readiness` CLI is now also exercised against the checked-in production example env files, and a spawned `website/server` bootstrap proof now verifies the live same-origin process path from production-shaped env into `/health`, `/api/auth/health`, built-shell serving, and the public anonymous release-manifest posture for the shared desktop release lane. The same verifier now also probes the deployed root-shell marker and can explicitly fail when the public origin is still serving the older placeholder rollout page instead of the first-party website/auth-server lane, while the repo now also carries first-party `website/deploy/` `nginx` plus `systemd` handoff templates, a concrete same-origin public-host cutover guide, a deployment-file renderer that emits resolved operator outputs from real checkout paths, and a manifest-driven bundle renderer that lets one authoritative input own the public origin while emitting validated env plus install artifacts together, with the shared-VPS-safe default upstream moved to `3011` after live host inspection confirmed `3001` is already occupied by FamiliarOS. The same deployment lane now also distinguishes `launch` from `preview` posture so honest missing checkout/download/webhook/release values are accepted only for non-public rehearsal while placeholder strings still fail, `runtime.mode: mixed` plus `public_origin_ready: true` counts as valid preview-host proof, and the staging helper can archive either committed `HEAD` or the live worktree through `--archive-source worktree`. An isolated VPS-local staging proof then confirmed that both the committed HyperTwist website lane and the later preview-tier worktree packet can serve green health, release-manifest, and first-party shell responses on that real shared host behind `3011`, and a later root-owned cutover then replaced the public placeholder site with the live first-party same-origin preview deployment on `https://hypertwist.app`. The package now also ships a bounded root-owned live-deploy helper that stages the committed checkout, uploads the rendered bundle, installs env, rebuilds the site, replaces the live `systemd` plus `nginx` files, and validates the public origin; that helper has already been re-proved idempotently against the live host. The repo now also ships that host-proof flow as a first-party staging helper so future sessions can rerun the temp checkout/build/boot verification path directly before or after root-owned cutover. The website lane now also owns a sanitized generated packaged-validation summary for the Windows higher-dimensional desktop proof, rendered from the checked-in authoritative package report into `website/src/shared/generated/windows-package-validation-summary.json`, and the owned web-surface validation gate now checks that generated proof for freshness before approving the current public/auth/download/browser lane. The repo now also carries checked-in preview-tier frontend/server env examples plus a matching preview same-origin bundle example for the current live `hypertwist.app` posture, while the launch-tier examples still intentionally fail until real checkout, download, source, and webhook values replace their placeholders. The remaining public conversion seams now also stop dead-ending on generic sign-in/support detours when the next lane is already known: the explorer CTA preserves `/register?next=/app`, operator fallback opens the protected downloads lane, studio fallback opens the protected browser-access rollout lane, and launch/download fallback callouts now point directly at protected release, notices, or dashboard follow-through. The narrative/help surfaces then widened again so the public About and Support pages also surface the same live launch-readiness callout and packaged desktop-proof section already used by the stronger public manual routes, keeping maturity language attached to current native release evidence instead of leaving those pages purely abstract. The same public manual continuation now also adds a direct reusable surface-choice guide across the homepage, About, Pricing, Download, and Support pages so operators no longer have to infer from the larger surface matrix alone when they should stay on the public web surface, move into the protected dashboard, or move into the packaged desktop runtime. A later same-family continuation then propagated that same guide through the remaining public feature/docs/resources/changelog/legal pages as well, so the whole public site now preserves one consistent browser-versus-protected-versus-desktop handoff story instead of leaving those lower-traffic surfaces on older implicit wording. A further same-day manual-clarity continuation then added a reusable direct comparison section across the homepage, About, Docs, and Download routes explaining why the website remains necessary, where it is intentionally narrower than the simulator, why the native Unreal runtime stays primary, and how the current keyboard or mouse versus unfinished XR/controller truth should be read. The latest same-family parity follow-up then widened that exact browser-versus-desktop reality section into Pricing and Resources as well, so commercial and reference-heavy public routes no longer rely only on the lighter surface-choice guide when explaining what stays native and where the current XR/controller boundary still begins. The next adjacent parity follow-up then extended the same direct comparison into the public Feature atlas and Support routes as well, so the remaining major public operator-facing surfaces no longer fall back to the lighter guide alone when capability review or recovery work needs the sharper website-versus-native split. A later same-family continuity follow-up then widened the shared release-decision guide into the remaining public legal/notices routes too, so open-source notices, privacy, terms, and shipping/payment now also tell the operator whether the next honest move is protected desktop access, pricing/provisioning, protected browser/account continuity, or notices/source follow-through instead of leaving those pages on the older lighter handoff only. The same browser shell now also has a first-party top-level runtime-recovery boundary, so unexpected React route failures degrade into a HyperTwist-owned retry/return surface that keeps the browser-versus-desktop product truth explicit instead of collapsing into a blank shell. A later same-family quickstart/manual continuation then added a reusable first serious-session guide across homepage, docs, and download so operators can see the real path from release-target choice through desktop-link pairing, first native verification, higher-dimensional verification, and the current XR/controller `No-Go` boundary in one place instead of reconstructing that flow from scattered adjacent sections. That same onboarding/manual packet is now also anchored at a dedicated public `/getting-started` route so the shortest complete browser-to-desktop first-session path is canonical, crawlable, and easier to hand off than the broader site sections alone. The next adjacent launch-authority continuation then did the same for rollout truth through a dedicated public `/launch-status` route, giving preview-versus-launch posture, rollout blockers, packaged proof, and release-reference follow-through one canonical public authority surface instead of leaving that lane distributed across pricing and compact status banners alone. The next same-family shared-auth manual continuation then widened the real browser-account method lineup beyond the auth entry forms and deeper docs surfaces into the homepage, pricing, download, and support routes too, so the public pages where operators actually decide to sign in, buy access, or recover access posture now expose the real provider lineup directly. The latest same-family manual continuation now also adds a shared public route atlas across homepage, docs, and resources so each major public page explains its exact job in the broader operator/distribution manual instead of leaving page purpose mostly implicit in navigation alone. The latest same-family atlas-parity follow-up then widened the public feature-atlas route again so it now also carries the shared runtime control guide directly, keeping the practical classic-control, higher-dimensional-control, native-diagnostics, and explicit XR/controller-boundary truth visible on `/features` instead of forcing advanced readers to leave the atlas for deeper manual routes. The latest same-family responsive-proof expansion then widened browser-level viewport evidence across those newer manual and authority routes too, so `/features`, `/docs`, `/getting-started`, and `/launch-status` now sit inside the owned public responsive validation lane instead of relying only on source-level or unit-level proof. The latest same-family commerce/download manual deepening follow-up then widened `/pricing` and `/download` again so the actual purchase and package routes now carry the installed-software simulator manual, the higher-dimensional family guide, and the current input/device plus runtime-control truth directly instead of leaving that practical product explanation to docs/resources/getting-started alone. A later same-family continuation then completed the shared-section path on the feature atlas itself by routing its higher-dimensional guide, input/device posture, control-profile roster, and runtime-control guide through the same shared helper ownership as the surrounding public routes, and widened responsive public-route proof across `/changelog`, `/open-source-notices`, `/privacy`, `/terms`, and `/shipping-payment`, while a later adjacent auth-entry parity follow-up also brought `/login` into that same owned mobile/tablet matrix, bringing the explicit public-route browser proof to `34` passed Playwright checks while the broader `13`-file/`79`-test website route/auth/release suite and `10`-file/`36`-test auth-server suite stayed green. | +| Public `hypertwist.app` marketing shell | Implemented now | first-party `website/` app + feature registry/roadmap authority | HyperTwist now has a dedicated first-party public web surface for homepage, feature atlas, about, resources, pricing, download, support, legal routes, a dedicated `/getting-started` onboarding surface, and a dedicated `/launch-status` authority surface. This lane is separate from the embedded Unreal browser runtime under `Content/Browser/` and does not claim browser-simulator parity. The same public lane now also serves as a bounded operator/distribution manual through the docs/resources/download/support surfaces, the public feature-atlas route, the dedicated getting-started route, the dedicated launch-status route, and related launch/status surfaces, explaining browser-versus-desktop posture, rollout steps, package proof, simulator-use guidance, and current input/device truth without claiming browser ownership of the native runtime or overclaiming unfinished VR/controller posture. The same package now also carries a first-party external runtime-readiness verifier so deploy-time env and live health posture can be checked outside the dashboard, plus separated local-versus-production env templates whose placeholder values are intentionally rejected until real launch config is in place, bootstrap CI now validates both the frontend and auth-server website commands directly, and the auth server can now auto-serve the built `website/dist` bundle with bounded SPA fallback for same-origin public deployment. Request-level server coverage now also proves that public/app shell delivery does not shadow `/api/*`, `/auth*`, `/health`, or missing asset paths, while the pricing/download/notices routes now surface first-party preview-versus-launch posture from the same bounded launch checklist instead of relying on hidden operator-only status. The shared marketing shell now also carries a compact public-site-status banner across public pages, the homepage keeps a fuller status section, and the shared public launch-status component now consumes live auth-health webhook/runtime truth in addition to release-manifest download readiness so public marketing copy does not claim launch posture from static checkout/download config alone. The auth server now also emits an authoritative `launch` summary on `/api/auth/health`, including required blocker labels, billing product/price-map gates, and current checkout targets, so the public banner and protected dashboard no longer duplicate launch-blocker evaluation client-side. A later same-family continuation then widened that release-manifest authority again to carry public runtime commerce config for operator/studio checkout URLs and live plan-price strings, so pricing, notices, and dashboard launch-readiness surfaces no longer depend only on frontend build-time checkout config. The live website lane now also owns route-aware title/description/canonical/Open-Graph/Twitter metadata for the real `hypertwist.app` marketing surface so deployed public pages no longer remain on a single generic SPA title/description, plus first-party `robots.txt` and `sitemap.xml` assets for the public route set while keeping `/app`, `/login`, and `/register` out of crawler posture. The later shared search-authority hardening pass then moved both assets onto one generated authority seam backed by `website/src/public-route-registry.json`, `website/src/public-search-policy.json`, and first-party script/test ownership, keeping `/app`, `/api/`, `/auth`, `/health`, `/login`, and `/register` aligned without hand-maintained crawler drift. The real `check-runtime-readiness` CLI is now also exercised against the checked-in production example env files, and a spawned `website/server` bootstrap proof now verifies the live same-origin process path from production-shaped env into `/health`, `/api/auth/health`, built-shell serving, and the public anonymous release-manifest posture for the shared desktop release lane. The same verifier now also probes the deployed root-shell marker and can explicitly fail when the public origin is still serving the older placeholder rollout page instead of the first-party website/auth-server lane, while the repo now also carries first-party `website/deploy/` `nginx` plus `systemd` handoff templates, a concrete same-origin public-host cutover guide, a deployment-file renderer that emits resolved operator outputs from real checkout paths, and a manifest-driven bundle renderer that lets one authoritative input own the public origin while emitting validated env plus install artifacts together, with the shared-VPS-safe default upstream moved to `3011` after live host inspection confirmed `3001` is already occupied by FamiliarOS. The same deployment lane now also distinguishes `launch` from `preview` posture so honest missing checkout/download/webhook/release values are accepted only for non-public rehearsal while placeholder strings still fail, `runtime.mode: mixed` plus `public_origin_ready: true` counts as valid preview-host proof, and the staging helper can archive either committed `HEAD` or the live worktree through `--archive-source worktree`. An isolated VPS-local staging proof then confirmed that both the committed HyperTwist website lane and the later preview-tier worktree packet can serve green health, release-manifest, and first-party shell responses on that real shared host behind `3011`, and a later root-owned cutover then replaced the public placeholder site with the live first-party same-origin preview deployment on `https://hypertwist.app`. The package now also ships a bounded root-owned live-deploy helper that stages the committed checkout, uploads the rendered bundle, installs env, rebuilds the site, replaces the live `systemd` plus `nginx` files, and validates the public origin; that helper has already been re-proved idempotently against the live host. The repo now also ships that host-proof flow as a first-party staging helper so future sessions can rerun the temp checkout/build/boot verification path directly before or after root-owned cutover. The website lane now also owns a sanitized generated packaged-validation summary for the Windows higher-dimensional desktop proof, rendered from the checked-in authoritative package report into `website/src/shared/generated/windows-package-validation-summary.json`, and the owned web-surface validation gate now checks that generated proof for freshness before approving the current public/auth/download/browser lane. The repo now also carries checked-in preview-tier frontend/server env examples plus a matching preview same-origin bundle example for the current live `hypertwist.app` posture, while the launch-tier examples still intentionally fail until real checkout, download, source, and webhook values replace their placeholders. The remaining public conversion seams now also stop dead-ending on generic sign-in/support detours when the next lane is already known: the explorer CTA preserves `/register?next=/app`, operator fallback opens the protected downloads lane, studio fallback opens the protected browser-access rollout lane, and launch/download fallback callouts now point directly at protected release, notices, or dashboard follow-through. The narrative/help surfaces then widened again so the public About and Support pages also surface the same live launch-readiness callout and packaged desktop-proof section already used by the stronger public manual routes, keeping maturity language attached to current native release evidence instead of leaving those pages purely abstract. The same public manual continuation now also adds a direct reusable surface-choice guide across the homepage, About, Pricing, Download, and Support pages so operators no longer have to infer from the larger surface matrix alone when they should stay on the public web surface, move into the protected dashboard, or move into the packaged desktop runtime. A later same-family continuation then propagated that same guide through the remaining public feature/docs/resources/changelog/legal pages as well, so the whole public site now preserves one consistent browser-versus-protected-versus-desktop handoff story instead of leaving those lower-traffic surfaces on older implicit wording. A further same-day manual-clarity continuation then added a reusable direct comparison section across the homepage, About, Docs, and Download routes explaining why the website remains necessary, where it is intentionally narrower than the simulator, why the native Unreal runtime stays primary, and how the current keyboard or mouse versus unfinished XR/controller truth should be read. The latest same-family parity follow-up then widened that exact browser-versus-desktop reality section into Pricing and Resources as well, so commercial and reference-heavy public routes no longer rely only on the lighter surface-choice guide when explaining what stays native and where the current XR/controller boundary still begins. The next adjacent parity follow-up then extended the same direct comparison into the public Feature atlas and Support routes as well, so the remaining major public operator-facing surfaces no longer fall back to the lighter guide alone when capability review or recovery work needs the sharper website-versus-native split. A later same-family continuity follow-up then widened the shared release-decision guide into the remaining public legal/notices routes too, so open-source notices, privacy, terms, and shipping/payment now also tell the operator whether the next honest move is protected desktop access, pricing/provisioning, protected browser/account continuity, or notices/source follow-through instead of leaving those pages on the older lighter handoff only. The same browser shell now also has a first-party top-level runtime-recovery boundary, so unexpected React route failures degrade into a HyperTwist-owned retry/return surface that keeps the browser-versus-desktop product truth explicit instead of collapsing into a blank shell. A later same-family quickstart/manual continuation then added a reusable first serious-session guide across homepage, docs, and download so operators can see the real path from release-target choice through desktop-link pairing, first native verification, higher-dimensional verification, and the current XR/controller `No-Go` boundary in one place instead of reconstructing that flow from scattered adjacent sections. That same onboarding/manual packet is now also anchored at a dedicated public `/getting-started` route so the shortest complete browser-to-desktop first-session path is canonical, crawlable, and easier to hand off than the broader site sections alone. The next adjacent launch-authority continuation then did the same for rollout truth through a dedicated public `/launch-status` route, giving preview-versus-launch posture, rollout blockers, packaged proof, and release-reference follow-through one canonical public authority surface instead of leaving that lane distributed across pricing and compact status banners alone. The next same-family shared-auth manual continuation then widened the real browser-account method lineup beyond the auth entry forms and deeper docs surfaces into the homepage, pricing, download, and support routes too, so the public pages where operators actually decide to sign in, buy access, or recover access posture now expose the real provider lineup directly. The latest same-family manual continuation now also adds a shared public route atlas across homepage, docs, and resources so each major public page explains its exact job in the broader operator/distribution manual instead of leaving page purpose mostly implicit in navigation alone. The latest same-family atlas-parity follow-up then widened the public feature-atlas route again so it now also carries the shared runtime control guide directly, keeping the practical classic-control, higher-dimensional-control, native-diagnostics, and explicit XR/controller-boundary truth visible on `/features` instead of forcing advanced readers to leave the atlas for deeper manual routes. The latest same-family responsive-proof expansion then widened browser-level viewport evidence across those newer manual and authority routes too, so `/features`, `/docs`, `/getting-started`, and `/launch-status` now sit inside the owned public responsive validation lane instead of relying only on source-level or unit-level proof. The latest same-family commerce/download manual deepening follow-up then widened `/pricing` and `/download` again so the actual purchase and package routes now carry the installed-software simulator manual, the higher-dimensional family guide, and the current input/device plus runtime-control truth directly instead of leaving that practical product explanation to docs/resources/getting-started alone. A later same-family continuation then completed the shared-section path on the feature atlas itself by routing its higher-dimensional guide, input/device posture, control-profile roster, and runtime-control guide through the same shared helper ownership as the surrounding public routes, and widened responsive public-route proof across `/changelog`, `/open-source-notices`, `/privacy`, `/terms`, and `/shipping-payment`, while a later adjacent auth-entry parity follow-up also brought `/login` into that same owned mobile/tablet matrix, bringing the explicit public-route browser proof to `34` passed Playwright checks while the broader `13`-file/`79`-test website route/auth/release suite and `10`-file/`36`-test auth-server suite stayed green. | | Browser-based operator/account dashboard | Implemented now | first-party `website/` app + shared auth/dashboard packet | A protected browser dashboard is now live for operator access, account state, download posture, browser-access boundary explanation, notices review, and bounded billing/entitlement status. It reuses the shared SuperTokens auth posture proven in FamiliarOS and ScriptoriumAI while remaining HyperTwist-specific in product content and boundary claims, the current auth-health surface now truthfully distinguishes configured versus reachable or ready shared-core posture while exposing fallback-active reason instead of hardcoding readiness, and the same dashboard now also surfaces launch-readiness truth for download URLs, checkout links, source/notices URLs, billing-secret/map configuration, and local-versus-public runtime deployment posture. The same shared-auth lane now also carries end-to-end optional provider parity for GitHub, Google, and bounded ORCID sign-in: the auth server owns the ORCID custom-provider path, the same-origin bundle emits matching ORCID frontend/server env truth, the browser client now registers ORCID through the supported third-party recipe provider contract instead of leaving that final step implicit, auth-health reports ORCID readiness alongside the other providers, and the public docs route now exposes the current browser-account method lineup instead of leaving provider truth visible only on the form pages. The surrounding public decision surfaces now also mirror that same provider-lineup truth on homepage, pricing, download, and support, so shared-auth reality is not hidden until operators reach either the auth forms or the deeper manual. The same protected overview now also consumes the server-backed Windows packaged-validation summary that the release-manifest authority exposes, so operators can see current higher-dimensional desktop proof without drilling into the dedicated downloads screen. The adjacent protected `/app/browser-access` and `/app/notices` routes now also reuse the bounded escalation-map and release-follow-through guidance from the public support/manual lane, so sign-in does not degrade operator troubleshooting quality into a thinner shell than the public surface. The protected overview and protected download center now also mirror the same launch-readiness, operator-access, and studio-rollout help lanes as the public support/manual surface, but translate those actions into direct signed-in dashboard, downloads, browser-access, account, notices, pricing, and release-notes targets instead of sending the operator back through anonymous auth detours. A later same-family parity follow-up then added the same blunt browser-versus-desktop reality explanation to the protected dashboard, protected downloads, and browser-access routes as well, so sign-in no longer makes the product boundary or unfinished XR/controller truth less explicit than the public manual. The next adjacent parity follow-up then extended that same compact reality panel into the protected account and notices routes too, so entitlement review and distribution/compliance work no longer soften the browser-support versus native-simulator split after sign-in. The auth entry pages now also render the same browser-versus-desktop truth before sign-in is even complete, so login and register no longer rely only on next-step routing plus later dashboard surfaces to explain what stays native and where the current XR/controller boundary still begins. A later same-family protected-shell continuation then widened signed-in auth truth itself, so the dashboard and account routes now surface the auth method, current provider lineup, shared-auth readiness, runtime origins, cookie/public-origin posture, fallback state, and deployment diagnostics directly instead of collapsing that truth down to a thin stack label. The protected fallback path now also preserves signed-in viewer plan/access posture from local session truth when live manifest authority is temporarily unavailable, while still keeping download authority conservative until the auth server returns. The shared browser auth layer now also has a bounded release-authority reconciliation path that refreshes and persists the local browser session snapshot from fresher manifest viewer truth, while the dashboard and account surfaces expose a compact live-authority sync notice summarizing what changed so operators can see the session catch-up clearly. That same auth layer now also clears stale stored sessions when shared auth is configured and `/api/auth/me` returns `401`, while still preserving the intended offline/local-fallback posture when the account API is merely unreachable or shared auth is not configured. The next adjacent protected-shell launch-authority continuation then added a dedicated signed-in `/app/launch-status` route, so launch posture, rollout blockers, release references, packaged proof, and signed-in next actions no longer collapse back to a single overview panel once account context exists. The follow-through continuation after that then threaded the same signed-in launch-status route back through protected release-action cards, release-reference panels, and the browser-access, download, account, and notices surfaces, so rollout authority remains attached to the surrounding signed-in operator shell instead of becoming an isolated page. Focused frontend coverage now also protects deep-link login redirect preservation, safe `next`-path normalization across auth entry points, fallback/email auth-bootstrap normalization, login/register continuation behavior, public download-gating behavior, protected-route/shell behavior, real lazy-route tree behavior for key public and protected paths, top-level app-bootstrap and SuperTokens-wrapper posture, login/register unhappy-path and OAuth-button behavior, support-topic fallback routing when live checkout is not configured, desktop-link verify-url/dashboard readiness behavior, and explicit `noindex,nofollow` posture on protected/auth browser surfaces. The validation lane now also has a bounded signed test-session harness under `TEST_MODE=testing` that proves `/api/auth/me`, stale-session invalidation, release-authority reconciliation, and `/api/auth/desktop-link` behavior through the live spawned auth-server process without widening production auth posture. A later same-family responsive-browser hardening follow-up then added Playwright viewport proof for `/app`, `/app/launch-status`, `/app/downloads`, `/app/browser-access`, `/app/account`, and `/app/notices`, so the shipped signed-in operator shell now has the same bounded mobile/tablet no-horizontal-overflow proof posture as the current public-route family. | | Desktop download posture and browser-to-desktop pairing | Implemented now | first-party `website/` app + `website/server` desktop-link endpoints | Public download targets, dashboard-side release posture, and short-lived desktop-link token generation/verification are now first-party owned. The current server posture now enforces exact website-origin matching, bounded per-user issuance, one-time token consumption, and billing-backed plan/download entitlement resolution with focused `website/server` tests green on `2026-06-22`, and the verify handshake now returns the same resolved download-entitlement posture the dashboard sees instead of only identity plus plan/role. The same lane now also owns a shared `GET /api/releases/manifest` runtime authority for release version/channel/build/published/file-size/checksum/docs/source metadata, with anonymous callers intentionally denied raw download URLs while entitled session-backed callers receive the configured direct platform URL. That manifest now also carries first-party packaged-validation summary truth for the Windows higher-dimensional desktop lane, so the public `/download` page, the public `/resources` reference page, and the protected `/app/downloads` surface can project real package evidence for the dedicated-family `Magic120Cell` / `MagicCube5D` maps even while launch-tier release URLs remain unconfigured. That website-facing packaged proof is now sourced through a sanitized generated summary rendered from the checked-in authoritative `phase6c_dedicated_family_package_validation_report.json`, and the owned web-surface validation gate now checks that generated summary for freshness before approving the current public/auth/download/browser lane. The public `/download` page now keeps raw download URLs behind the protected dashboard instead of exposing them directly, preserves requested platform continuity through `/app/downloads?platform=...`, and surfaces that requested target again after auth handoff inside the protected release lane. The public download fallbacks now also open exact protected download and dashboard follow-through instead of stopping at a generic anonymous login hop. When live manifest authority is unavailable, the protected fallback path now still reflects the signed-in viewer posture from local session truth while continuing to withhold raw delivery authority. When live manifest authority resolves fresher entitled viewer truth than the local browser auth snapshot, the protected download-center lane now also follows that server-backed viewer posture for the actual download action instead of leaving the operator blocked behind stale local session state. Both the public and protected download surfaces now also carry first-party rollout steps, first-launch desktop setup guidance, browser-to-desktop pairing follow-through, and release/notices/source references so the desktop setup lane is more than a generic link bucket, while the support surface now carries an explicit escalation map separating account, package, runtime, and rollout/compliance problems. The protected dashboard overview and protected account surface now also mirror that same first-launch follow-through, and the protected browser-access/notices routes now mirror the same escalation separation, so post-sign-in operator guidance does not collapse back into a thinner release-only shell. The protected download center now also mirrors the same three help-topic lanes used by the public support/manual surface, but with direct signed-in actions for downloads, dashboard, account, browser access, notices, and release notes so entitled operators can stay inside the protected rollout lane once auth has already succeeded. The dashboard plus public launch-status callouts now consume the same manifest-backed Windows download truth instead of only static frontend config. Actual release URLs remain deployment configuration rather than hardcoded product truth. | | Paddle-ready pricing and billing webhook seam | Implemented now | first-party `website/` app + `website/server` billing endpoint | The public pricing surface now exists with plan structure, checkout-link configuration seams, and the same `/api/billing/paddle/webhook` endpoint family used by the broader product website lane. The current server now verifies `Paddle-Signature` against `PADDLE_WEBHOOK_SECRET` using the documented raw-body HMAC flow, persists a bounded first-party billing state file, and applies verified Paddle events into account/download entitlement state that the browser dashboard consumes, with focused `website/server` tests green on `2026-06-22`. The shared `GET /api/releases/manifest` authority now also carries public runtime commerce config for operator/studio checkout URLs and live plan-price strings, allowing the pricing page to switch from frontend build-time checkout assumptions to auth-server runtime truth when those values are configured. The fallback plan CTAs now also use exact protected release/browser-access targets when live checkout URLs are absent, instead of routing operators back into a vague public support detour. A spawned live-process proof now also verifies that a real signed webhook updates processed-event health and persisted billing state through the actual auth-server runtime, not only helper-level store tests, and transaction events no longer leak their id into stored `subscriptionId` state. Production checkout URLs, secret management, and broader operator/admin billing workflows remain deployment/application tasks, not shipped-code omissions. | diff --git a/scripts/run-hypertwist-web-surface-validation.sh b/scripts/run-hypertwist-web-surface-validation.sh index d63bcb6..db548a2 100644 --- a/scripts/run-hypertwist-web-surface-validation.sh +++ b/scripts/run-hypertwist-web-surface-validation.sh @@ -166,6 +166,7 @@ run_step \ run_step \ "website deployment/readiness tooling validation" \ npm --prefix website test -- --run \ + scripts/render-public-search-assets-lib.test.mjs \ scripts/runtime-readiness-lib.test.mjs \ scripts/runtime-readiness-cli.test.mjs \ scripts/render-same-origin-bundle-lib.test.mjs diff --git a/website/package.json b/website/package.json index 9071433..42f6576 100644 --- a/website/package.json +++ b/website/package.json @@ -6,7 +6,7 @@ "type": "module", "scripts": { "dev": "vite", - "build": "npm run render:sitemap && tsc && vite build", + "build": "npm run render:public-search-assets && tsc && vite build", "preview": "vite preview", "type-check": "tsc --noEmit", "test": "vitest run", @@ -14,6 +14,7 @@ "test:e2e:responsive:list": "playwright test tests/e2e/responsive-public-pages.spec.ts --list", "test:e2e:protected-responsive": "playwright test tests/e2e/responsive-protected-app-routes.spec.ts --reporter=line", "test:e2e:protected-responsive:list": "playwright test tests/e2e/responsive-protected-app-routes.spec.ts --list", + "render:public-search-assets": "node scripts/render-public-search-assets.mjs", "render:sitemap": "node scripts/render-public-sitemap.mjs", "check:runtime-readiness": "node scripts/check-runtime-readiness.mjs", "check:runtime-readiness:preview-live": "node scripts/check-runtime-readiness.mjs --frontend-env .env.preview.example --server-env server/.env.preview.example --health-url https://hypertwist.app --deployment-tier preview", diff --git a/website/public/robots.txt b/website/public/robots.txt index 48e7bbb..a15a70f 100644 --- a/website/public/robots.txt +++ b/website/public/robots.txt @@ -1,6 +1,9 @@ User-agent: * Allow: / Disallow: /app +Disallow: /api/ +Disallow: /auth +Disallow: /health Disallow: /login Disallow: /register diff --git a/website/scripts/render-public-search-assets-lib.mjs b/website/scripts/render-public-search-assets-lib.mjs new file mode 100644 index 0000000..3e75822 --- /dev/null +++ b/website/scripts/render-public-search-assets-lib.mjs @@ -0,0 +1,106 @@ +import { readFile, writeFile } from 'node:fs/promises' + +function normalizeBaseUrl(baseUrl) { + return String(baseUrl || '').trim().replace(/\/$/, '') || 'https://hypertwist.app' +} + +function uniqueTrimmedStrings(values) { + const result = [] + const seen = new Set() + + for (const rawValue of Array.isArray(values) ? values : []) { + const value = String(rawValue || '').trim() + if (!value || seen.has(value)) { + continue + } + + seen.add(value) + result.push(value) + } + + return result +} + +function readRoutePath(route) { + const path = String(route?.path || '').trim() + return path || null +} + +export function buildPublicSearchAssetPlan(routeRegistry, policy = {}) { + const sanitizedRoutes = Array.isArray(routeRegistry) ? routeRegistry.filter((route) => readRoutePath(route)) : [] + const crawlableRoutes = sanitizedRoutes.filter((route) => route?.crawlable === true) + const nonCrawlablePaths = sanitizedRoutes + .filter((route) => route?.crawlable !== true) + .map((route) => readRoutePath(route)) + .filter(Boolean) + + const explicitDisallowPaths = uniqueTrimmedStrings(policy?.explicitDisallowPaths) + const crawlerDisallowPaths = uniqueTrimmedStrings([ + ...explicitDisallowPaths, + ...nonCrawlablePaths, + ]) + + return { + baseUrl: normalizeBaseUrl(policy?.baseUrl), + crawlableRoutes, + crawlerDisallowPaths, + } +} + +export function buildPublicSitemapXml(routeRegistry, policy = {}) { + const { baseUrl, crawlableRoutes } = buildPublicSearchAssetPlan(routeRegistry, policy) + const urlEntries = crawlableRoutes + .map((route) => ` \n ${baseUrl}${route.path === '/' ? '/' : route.path}\n `) + .join('\n') + + return `\n\n${urlEntries}\n\n` +} + +export function buildPublicRobotsTxt(routeRegistry, policy = {}) { + const { baseUrl, crawlerDisallowPaths } = buildPublicSearchAssetPlan(routeRegistry, policy) + + return [ + 'User-agent: *', + 'Allow: /', + ...crawlerDisallowPaths.map((path) => `Disallow: ${path}`), + '', + `Sitemap: ${baseUrl}/sitemap.xml`, + '', + ].join('\n') +} + +export async function loadPublicSearchAssetInputs({ registryPath, policyPath }) { + const [routeRegistry, policy] = await Promise.all([ + readFile(registryPath, 'utf8').then((content) => JSON.parse(content)), + readFile(policyPath, 'utf8').then((content) => JSON.parse(content)), + ]) + + return { routeRegistry, policy } +} + +export async function renderPublicSearchAssets({ + registryPath, + policyPath, + sitemapPath, + robotsPath, +}) { + const { routeRegistry, policy } = await loadPublicSearchAssetInputs({ + registryPath, + policyPath, + }) + + const plan = buildPublicSearchAssetPlan(routeRegistry, policy) + const sitemapXml = buildPublicSitemapXml(routeRegistry, policy) + const robotsTxt = buildPublicRobotsTxt(routeRegistry, policy) + + await Promise.all([ + writeFile(sitemapPath, sitemapXml, 'utf8'), + writeFile(robotsPath, robotsTxt, 'utf8'), + ]) + + return { + ...plan, + sitemapXml, + robotsTxt, + } +} diff --git a/website/scripts/render-public-search-assets-lib.test.mjs b/website/scripts/render-public-search-assets-lib.test.mjs new file mode 100644 index 0000000..3fed503 --- /dev/null +++ b/website/scripts/render-public-search-assets-lib.test.mjs @@ -0,0 +1,104 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import { describe, expect, it } from 'vitest' + +import { + buildPublicRobotsTxt, + buildPublicSearchAssetPlan, + buildPublicSitemapXml, + renderPublicSearchAssets, +} from './render-public-search-assets-lib.mjs' + +function createRouteRegistry() { + return [ + { path: '/', crawlable: true }, + { path: '/features', crawlable: true }, + { path: '/login', crawlable: false }, + { path: '/register', crawlable: false }, + ] +} + +function createPolicy() { + return { + baseUrl: 'https://hypertwist.app/', + explicitDisallowPaths: ['/app', '/api/', '/auth', '/health', '/app'], + } +} + +describe('buildPublicSearchAssetPlan', () => { + it('derives crawlable routes and crawler disallow rules from shared inputs', () => { + const plan = buildPublicSearchAssetPlan(createRouteRegistry(), createPolicy()) + + expect(plan.baseUrl).toBe('https://hypertwist.app') + expect(plan.crawlableRoutes.map((route) => route.path)).toEqual(['/', '/features']) + expect(plan.crawlerDisallowPaths).toEqual([ + '/app', + '/api/', + '/auth', + '/health', + '/login', + '/register', + ]) + }) +}) + +describe('buildPublicSitemapXml', () => { + it('renders crawlable routes into sitemap XML', () => { + const xml = buildPublicSitemapXml(createRouteRegistry(), createPolicy()) + + expect(xml).toContain('https://hypertwist.app/') + expect(xml).toContain('https://hypertwist.app/features') + expect(xml).not.toContain('/login') + expect(xml).not.toContain('/register') + }) +}) + +describe('buildPublicRobotsTxt', () => { + it('renders the shared protected-route boundary into robots posture', () => { + const robots = buildPublicRobotsTxt(createRouteRegistry(), createPolicy()) + + expect(robots).toContain('User-agent: *') + expect(robots).toContain('Allow: /') + expect(robots).toContain('Disallow: /app') + expect(robots).toContain('Disallow: /api/') + expect(robots).toContain('Disallow: /auth') + expect(robots).toContain('Disallow: /health') + expect(robots).toContain('Disallow: /login') + expect(robots).toContain('Disallow: /register') + expect(robots).toContain('Sitemap: https://hypertwist.app/sitemap.xml') + }) +}) + +describe('renderPublicSearchAssets', () => { + it('writes both sitemap and robots assets from the same authority inputs', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hypertwist-public-search-assets-')) + const registryPath = path.join(tempRoot, 'public-route-registry.json') + const policyPath = path.join(tempRoot, 'public-search-policy.json') + const sitemapPath = path.join(tempRoot, 'sitemap.xml') + const robotsPath = path.join(tempRoot, 'robots.txt') + + fs.writeFileSync(registryPath, JSON.stringify(createRouteRegistry(), null, 2)) + fs.writeFileSync(policyPath, JSON.stringify(createPolicy(), null, 2)) + + const rendered = await renderPublicSearchAssets({ + registryPath, + policyPath, + sitemapPath, + robotsPath, + }) + + expect(rendered.crawlableRoutes).toHaveLength(2) + expect(rendered.crawlerDisallowPaths).toEqual([ + '/app', + '/api/', + '/auth', + '/health', + '/login', + '/register', + ]) + expect(fs.readFileSync(sitemapPath, 'utf8')).toContain('https://hypertwist.app/features') + expect(fs.readFileSync(robotsPath, 'utf8')).toContain('Disallow: /health') + }) +}) diff --git a/website/scripts/render-public-search-assets.mjs b/website/scripts/render-public-search-assets.mjs new file mode 100644 index 0000000..807a038 --- /dev/null +++ b/website/scripts/render-public-search-assets.mjs @@ -0,0 +1,22 @@ +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { renderPublicSearchAssets } from './render-public-search-assets-lib.mjs' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +const registryPath = path.join(__dirname, '..', 'src', 'public-route-registry.json') +const policyPath = path.join(__dirname, '..', 'src', 'public-search-policy.json') +const sitemapPath = path.join(__dirname, '..', 'public', 'sitemap.xml') +const robotsPath = path.join(__dirname, '..', 'public', 'robots.txt') + +const rendered = await renderPublicSearchAssets({ + registryPath, + policyPath, + sitemapPath, + robotsPath, +}) + +console.log( + `Rendered public search assets for ${rendered.crawlableRoutes.length} crawlable routes and ${rendered.crawlerDisallowPaths.length} crawler disallow rules.`, +) diff --git a/website/scripts/render-public-sitemap.mjs b/website/scripts/render-public-sitemap.mjs index b92257b..ec82e37 100644 --- a/website/scripts/render-public-sitemap.mjs +++ b/website/scripts/render-public-sitemap.mjs @@ -1,24 +1 @@ -import { readFile, writeFile } from 'node:fs/promises' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const registryPath = path.join(__dirname, '..', 'src', 'public-route-registry.json') -const sitemapPath = path.join(__dirname, '..', 'public', 'sitemap.xml') -const baseUrl = 'https://hypertwist.app' - -const routeRegistry = JSON.parse(await readFile(registryPath, 'utf8')) -const crawlableRoutes = routeRegistry.filter((route) => route && route.crawlable === true) - -const xml = [ - '', - '', - ...crawlableRoutes.map((route) => ( - ` \n ${baseUrl}${route.path === '/' ? '/' : route.path}\n ` - )), - '', - '', -].join('\n') - -await writeFile(sitemapPath, xml, 'utf8') -console.log(`Rendered sitemap for ${crawlableRoutes.length} crawlable routes to ${sitemapPath}`) +import './render-public-search-assets.mjs' diff --git a/website/src/__tests__/public-route-registry.test.ts b/website/src/__tests__/public-route-registry.test.ts index 427bbf5..a9cc5ff 100644 --- a/website/src/__tests__/public-route-registry.test.ts +++ b/website/src/__tests__/public-route-registry.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest' import { + buildPublicRobotsTxt, buildPublicSitemapXml, crawlablePublicRoutes, footerLinks, marketingNavLinks, + publicCrawlerDisallowPaths, publicRouteRegistry, } from '../public-route-registry' @@ -37,6 +39,17 @@ describe('public route registry', () => { expect(crawlablePublicRoutes.map((route) => route.path)).toContain('/launch-status') }) + it('keeps protected runtime and auth boundaries out of crawler posture', () => { + expect(publicCrawlerDisallowPaths).toEqual([ + '/app', + '/api/', + '/auth', + '/health', + '/login', + '/register', + ]) + }) + it('carries non-empty loader metadata for every shared public-shell route', () => { for (const route of publicRouteRegistry) { expect(route.loaderTitle.trim().length).toBeGreaterThan(0) @@ -55,4 +68,19 @@ describe('public route registry', () => { expect(xml).not.toContain('/login') expect(xml).not.toContain('/register') }) + + it('renders robots.txt from the shared crawler policy and route registry', () => { + const robots = buildPublicRobotsTxt() + + expect(robots).toContain('User-agent: *') + expect(robots).toContain('Allow: /') + expect(robots).toContain('Disallow: /app') + expect(robots).toContain('Disallow: /api/') + expect(robots).toContain('Disallow: /auth') + expect(robots).toContain('Disallow: /health') + expect(robots).toContain('Disallow: /login') + expect(robots).toContain('Disallow: /register') + expect(robots).toContain('Sitemap: https://hypertwist.app/sitemap.xml') + expect(robots).not.toContain('Disallow: /features') + }) }) diff --git a/website/src/public-route-registry.ts b/website/src/public-route-registry.ts index 39b2c08..a5fcf4d 100644 --- a/website/src/public-route-registry.ts +++ b/website/src/public-route-registry.ts @@ -1,4 +1,5 @@ import routeRegistry from './public-route-registry.json' +import publicSearchPolicy from './public-search-policy.json' export type PublicRouteDefinition = { path: string @@ -11,7 +12,30 @@ export type PublicRouteDefinition = { crawlable: boolean } +type PublicSearchPolicyDefinition = { + baseUrl: string + explicitDisallowPaths: string[] +} + export const publicRouteRegistry = routeRegistry as PublicRouteDefinition[] +export const searchCrawlerPolicy = publicSearchPolicy as PublicSearchPolicyDefinition + +function uniqueTrimmedPaths(paths: readonly string[]) { + const result: string[] = [] + const seen = new Set() + + for (const rawPath of paths) { + const path = String(rawPath || '').trim() + if (!path || seen.has(path)) { + continue + } + + seen.add(path) + result.push(path) + } + + return result +} export const marketingNavLinks = publicRouteRegistry .filter((route) => route.nav) @@ -28,8 +52,14 @@ export const footerLinks = publicRouteRegistry })) export const crawlablePublicRoutes = publicRouteRegistry.filter((route) => route.crawlable) +export const nonCrawlablePublicRoutes = publicRouteRegistry.filter((route) => !route.crawlable) +export const publicSearchBaseUrl = String(searchCrawlerPolicy.baseUrl || '').trim().replace(/\/$/, '') || 'https://hypertwist.app' +export const publicCrawlerDisallowPaths = uniqueTrimmedPaths([ + ...(searchCrawlerPolicy.explicitDisallowPaths || []), + ...nonCrawlablePublicRoutes.map((route) => route.path), +]) -export function buildPublicSitemapXml(baseUrl = 'https://hypertwist.app') { +export function buildPublicSitemapXml(baseUrl = publicSearchBaseUrl) { const normalizedBaseUrl = baseUrl.replace(/\/$/, '') const urlEntries = crawlablePublicRoutes .map((route) => ` \n ${normalizedBaseUrl}${route.path === '/' ? '/' : route.path}\n `) @@ -37,3 +67,16 @@ export function buildPublicSitemapXml(baseUrl = 'https://hypertwist.app') { return `\n\n${urlEntries}\n\n` } + +export function buildPublicRobotsTxt(baseUrl = publicSearchBaseUrl) { + const normalizedBaseUrl = baseUrl.replace(/\/$/, '') + + return [ + 'User-agent: *', + 'Allow: /', + ...publicCrawlerDisallowPaths.map((path) => `Disallow: ${path}`), + '', + `Sitemap: ${normalizedBaseUrl}/sitemap.xml`, + '', + ].join('\n') +} diff --git a/website/src/public-search-policy.json b/website/src/public-search-policy.json new file mode 100644 index 0000000..dec59e4 --- /dev/null +++ b/website/src/public-search-policy.json @@ -0,0 +1,9 @@ +{ + "baseUrl": "https://hypertwist.app", + "explicitDisallowPaths": [ + "/app", + "/api/", + "/auth", + "/health" + ] +}