diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12e58c9c0..653159465 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,34 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major
## [Unreleased]
-Nothing yet.
+### Added
+
+- **cli:** the web role serves `GET /robots.txt` and `GET /sitemap.xml` in `x dev` and in
+ `runRole`. Until now only the static export wrote them: a `ROLE=web` container answered 404 for
+ both, and an app has no way to add a non-page GET route. Both come from the new `siteSeo()`
+ (exported from `@ultimat3/cli`): the public `site/` routes (no `policy`), minus any page whose
+ `meta` says `robots: { index: false }`, dynamic routes expanded through `prerender()`. URLs are
+ absolute against `APP_URL`, else `SITE_ORIGIN`, else the request's origin. Production allows the
+ crawl and names the absolute sitemap; anything else is `Disallow: /`. The scaffolded
+ `apps/web/prerender.ts` now writes `siteSeo()`'s files, so the export and the process serve the
+ same two files. An app scaffolded earlier keeps its own `writeSeoFiles`; replace its body with
+ `siteSeo({ baseUrl, pagesFor })` to match. Past 50,000 URLs the web role serves the index but
+ not the `/sitemap-N.xml` parts.
+
+### Fixed
+
+- **cli:** with `realtime: { enabled: false }`, documents no longer carry
+ ``, the sync-worker meta or the page-boot script, and the worker and
+ boot routes are not mounted. No sync node is started in that case, so the page runtime was handed
+ a target nothing served.
+- **cli:** `x g job` / `x g task` no longer corrupt `apps/web/api/index.ts` when its `jobs:` or
+ `tasks:` list is already one entry per line, which is the shape the generator itself writes once
+ a list passes 100 columns. The rewrite nested a second `jobs: [` inside the first and left the old
+ `]` behind. Lists are now also searched only inside the `defineApi({` call.
+- **admin:** the `/_x` dev dashboard renders its tabs and questions in the framework's own locale.
+ In an app whose default locale is not `en`, every tab read `⟦dev.panel.mail.title⟧`. The
+ framework catalog is registered under `en` only, and the shell translated through the app's
+ ambient locale.
## 22.2.1 - 2026-09-25
diff --git a/packages/admin/src/dev/server.test.ts b/packages/admin/src/dev/server.test.ts
index cd0835101..e4585a61d 100644
--- a/packages/admin/src/dev/server.test.ts
+++ b/packages/admin/src/dev/server.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, test } from 'bun:test';
+import { configureLocales, resetLocaleConfig } from '@ultimat3/i18n';
import { staticDevSources } from './data';
import { assertDevOnly, DEV_PANELS, devDashboard, devShellStyle } from './server';
@@ -170,3 +171,22 @@ describe('devShellStyle', () => {
expect(await html?.text()).toContain(``);
});
});
+
+// Reported by an app whose default locale is `es-co`: every tab and question on /_x read
+// `⟦dev.panel.mail.title⟧`. The framework's strings are registered under `en` only, on purpose, and
+// the shell rendered through the AMBIENT locale — the app's. /_x is the framework's own English
+// tool (``), so it reads the framework catalog in its own locale.
+describe('/_x in an app whose locale is not en', () => {
+ test('every tab and the question render as text, never as a missing key', async () => {
+ configureLocales({ supported: ['es-co', 'en'], fallback: 'es-co' });
+ try {
+ const dashboard = devDashboard({ role: 'web', env: 'development', sources });
+ const html = await (await dashboard.handle(new Request('http://x/_x/mail')))?.text();
+ expect(html).toContain('>Mail');
+ expect(html).toContain('what did that email look like, in that locale?');
+ expect(html).not.toContain('⟦');
+ } finally {
+ resetLocaleConfig();
+ }
+ });
+});
diff --git a/packages/admin/src/dev/server.ts b/packages/admin/src/dev/server.ts
index edd5b1dab..98628eb5c 100644
--- a/packages/admin/src/dev/server.ts
+++ b/packages/admin/src/dev/server.ts
@@ -5,7 +5,7 @@
// Type-only, so it is erased and the 46-component barrel stays out of the mount graph — the
// values arrive through the dynamic `import()` in `devShellStyle()`, same reason as `data.ts`.
import { DEFAULT_ENVIRONMENT, tryResolveEnvironment } from '@ultimat3/core';
-import { t } from '@ultimat3/i18n';
+import { FRAMEWORK_CATALOG_LOCALE, translatorFor } from '@ultimat3/i18n';
import type { ColorRole } from '@ultimat3/ui';
import { DevDashboardInProdError } from '../errors';
import { defaultDevSources } from './data';
@@ -157,6 +157,13 @@ html[data-theme="dark"] { ${block('dark')} }
${SHELL_LAYOUT}`;
}
+/**
+ * /_x is the framework's own tool, in the framework's own locale (`` below): its
+ * strings are registered under `FRAMEWORK_CATALOG_LOCALE` only, so the AMBIENT locale — the app's,
+ * `es-co` for an app that defaults to it — answered `⟦dev.panel.mail.title⟧` for every tab.
+ */
+const t = (key: string): string => translatorFor(FRAMEWORK_CATALOG_LOCALE)(key);
+
function shell(
style: string,
basePath: string,
diff --git a/packages/cli/src/api-registration.test.ts b/packages/cli/src/api-registration.test.ts
index bdec705ce..ddbcc1245 100644
--- a/packages/cli/src/api-registration.test.ts
+++ b/packages/cli/src/api-registration.test.ts
@@ -54,3 +54,79 @@ describe('unit · inserting into the scaffolded index', () => {
expect(insertApiEntries(foreign, entries)).toEqual({ source: foreign, skipped: entries });
});
});
+
+// Reproduced from an app: `x g task --feature ` writes the task AND its job, and the
+// app's `jobs:` list was already one entry per line — which is the shape this edit itself produces
+// once a list passes 100 columns. The line-start lookup landed on the first ITEM rather than on
+// `jobs: [`, so the rewrite nested a second `jobs: [` inside the first and orphaned its `]`.
+describe('unit · a list already wrapped one entry per line', () => {
+ const WRAPPED = [
+ "import { defineApi } from '@ultimat3/action';",
+ "import * as anchorDayJob from '../app/evidence/jobs/anchor-day-job';",
+ "import * as anchorDay from '../app/evidence/tasks/anchor-day';",
+ "import * as health from './health';",
+ '',
+ 'export const api = defineApi({',
+ ' // Every primitive module under apps/web/app/*/{actions,queries,live,jobs,tasks}/, one entry each.',
+ ' actions: [',
+ ' health,',
+ ' ],',
+ ' jobs: [',
+ ' anchorDayJob,',
+ ' expireCreditsJob,',
+ ' ],',
+ ' tasks: [',
+ ' anchorDay,',
+ ' ],',
+ '});',
+ '',
+ ].join('\n');
+
+ test('a task and its job land in their lists, each list still one well-formed literal', () => {
+ const entries = apiEntriesFor([
+ 'apps/web/app/billing/tasks/purge-drafts.ts',
+ 'apps/web/app/billing/jobs/purge-drafts-job.ts',
+ ]);
+ const { source, skipped } = insertApiEntries(WRAPPED, entries);
+
+ expect(skipped).toEqual([]);
+ expect(source.match(/jobs: \[/g)).toHaveLength(1);
+ expect(source.match(/tasks: \[/g)).toHaveLength(1);
+ expect(source).toContain(' jobs: [anchorDayJob, expireCreditsJob, purgeDraftsJob],\n');
+ expect(source).toContain(' tasks: [anchorDay, purgeDrafts],\n');
+ expect(source).toContain(' actions: [\n health,\n ],\n');
+ expect(source.endsWith('});\n')).toBe(true);
+ // And it stays that way: the next run finds both names and changes nothing.
+ expect(insertApiEntries(source, entries).source).toBe(source);
+ });
+
+ // The scaffold's own index, grown one job at a time past the 100-column wrap — the shape every
+ // app reaches on its own, with nothing but this generator writing the list.
+ test('growing the scaffold index one job at a time keeps one jobs list, every job in it', () => {
+ const names = [
+ 'alpha',
+ 'bravo',
+ 'charlie',
+ 'delta',
+ 'echo',
+ 'foxtrot',
+ 'golf',
+ 'hotel',
+ 'india',
+ 'juliet',
+ 'kilo',
+ 'lima',
+ ];
+ let source = indexOf(true);
+ for (const name of names) {
+ source = insertApiEntries(
+ source,
+ apiEntriesFor([`apps/web/app/${name}-slice/jobs/${name}-job.ts`]),
+ ).source;
+ }
+ expect(source.match(/jobs: \[/g)).toHaveLength(1);
+ const list = /jobs: \[(?[^\]]*)\]/.exec(source)?.groups?.['body'] ?? '';
+ for (const name of names) expect(list).toContain(`${name}Job`);
+ expect(list).toContain('reindexPost');
+ });
+});
diff --git a/packages/cli/src/api-registration.ts b/packages/cli/src/api-registration.ts
index e374f55c7..96dff43cf 100644
--- a/packages/cli/src/api-registration.ts
+++ b/packages/cli/src/api-registration.ts
@@ -34,14 +34,27 @@ export function apiEntriesFor(written: readonly string[]): readonly ApiEntry[] {
});
}
-/** Every `[...]` entry of a `key: [...]` list in the `defineApi({ ... })` call. */
+/**
+ * Every `[...]` entry of a `key: [...]` list in the `defineApi({ ... })` call, and where it sits:
+ * `line` is the start of the line holding `key: [`, `start` is just past the `[`, `end` is the `]`.
+ *
+ * Searched from `from` (the `defineApi({` call), so a `jobs: [` in a comment or an object above the
+ * call is never the one edited. `line` is found from the key, never from `start`: in a list already
+ * wrapped one entry per line the character AT `start` is the newline after `[`, and a backwards
+ * search from there answered the first ITEM's line — the rewrite then nested a second `jobs: [`
+ * inside the first and left the old `]` behind.
+ */
const listOf = (
source: string,
key: string,
-): { start: number; end: number; items: string[] } | undefined => {
- const open = new RegExp(`\\n(\\s*)${key}: \\[`).exec(source);
- if (open === null) return undefined;
- const start = open.index + open[0].length;
+ from: number,
+): { line: number; start: number; end: number; items: string[] } | undefined => {
+ const open = new RegExp(`\\n([ \\t]*)${key}: \\[`, 'g');
+ open.lastIndex = from;
+ const found = open.exec(source);
+ if (found === null) return undefined;
+ const line = found.index + 1;
+ const start = found.index + found[0].length;
const end = source.indexOf(']', start);
if (end === -1) return undefined;
const items = source
@@ -49,7 +62,7 @@ const listOf = (
.split(',')
.map((item) => item.trim())
.filter((item) => item.length > 0);
- return { start, end, items };
+ return { line, start, end, items };
};
/**
@@ -68,12 +81,12 @@ export function insertApiEntries(
for (const entry of entries) {
const importLine = `import * as ${entry.binding} from '${entry.specifier}';`;
const call = next.indexOf('defineApi({');
- const actions = listOf(next, 'actions');
+ const actions = call === -1 ? undefined : listOf(next, 'actions', call);
if (call === -1 || actions === undefined) {
skipped.push(entry);
continue;
}
- const list = listOf(next, entry.key);
+ const list = listOf(next, entry.key, call);
if (list?.items.includes(entry.binding) === true) continue;
if (list === undefined) {
// After `actions: [...],` — the order `x new` writes: actions, queries, jobs, tasks.
@@ -82,10 +95,9 @@ export function insertApiEntries(
next = `${next.slice(0, after + 1)}${line}\n${next.slice(after + 1)}`;
} else {
const items = [...list.items, entry.binding];
- const lineStart = next.lastIndexOf('\n', list.start) + 1;
- const indent = /^\s*/.exec(next.slice(lineStart))?.[0] ?? '';
+ const indent = /^[ \t]*/.exec(next.slice(list.line))?.[0] ?? '';
const rewritten = wrapList(indent, `${entry.key}: [`, items, ']');
- next = `${next.slice(0, lineStart)}${rewritten}${next.slice(list.end + 1)}`;
+ next = `${next.slice(0, list.line)}${rewritten}${next.slice(list.end + 1)}`;
}
if (!next.includes(importLine)) next = withImport(next, importLine);
}
diff --git a/packages/cli/src/cmd-dev.ts b/packages/cli/src/cmd-dev.ts
index 089b42961..81ecee03f 100644
--- a/packages/cli/src/cmd-dev.ts
+++ b/packages/cli/src/cmd-dev.ts
@@ -223,6 +223,7 @@ async function bootDev(
storage: runtime.storage,
dashboard,
islands: () => state.islands,
+ realtime: runtime.realtime,
});
// The app's `apps//runtime.ts`, composed exactly as `runRole` composes a caller's
diff --git a/packages/cli/src/cmd-new.test.ts b/packages/cli/src/cmd-new.test.ts
index 431fcdf8f..8f3381eb7 100644
--- a/packages/cli/src/cmd-new.test.ts
+++ b/packages/cli/src/cmd-new.test.ts
@@ -404,15 +404,13 @@ describe('unit · x new · the API surface registers the app own primitives by n
* to answer — and `default: true` is a field only `--json` renders.
*/
describe('unit · x new · the sitemap and robots.txt it wires', () => {
- // Decision D2: `@ultimat3/seo` had eleven exports with zero callers anywhere, and `buildSitemap`
- // / `buildRobots` are the two the framework has no live equivalent for. They are wired into the
- // STATIC ENTRY rather than into a route because both belong to the artifact: a static export is
- // served with no process behind it, so a route answering them is a file the CDN never has.
- test('the static entry builds both, and the app declares the package it builds them with', () => {
+ // Decision D2 wired `buildSitemap` / `buildRobots` into the STATIC ENTRY, because a static export
+ // is served with no process behind it. A container is served WITH one, and answered 404 for both
+ // until 22.2.2: the entry now writes `siteSeo`'s answer, the same one the web role serves.
+ test('the static entry writes both, from the answer the web role serves', () => {
const entry = emitted('apps/web/prerender.ts', true);
- expect(entry).toContain("from '@ultimat3/seo'");
- expect(entry).toContain('buildSitemap(');
- expect(entry).toContain('buildRobots(');
+ expect(entry).toContain("from '@ultimat3/cli'");
+ expect(entry).toContain('siteSeo(');
expect(entry).toContain("'robots.txt'");
expect(emitted('package.json', true)).toContain('"@ultimat3/seo"');
});
diff --git a/packages/cli/src/dev-route-table.ts b/packages/cli/src/dev-route-table.ts
index 42293b50b..212534531 100644
--- a/packages/cli/src/dev-route-table.ts
+++ b/packages/cli/src/dev-route-table.ts
@@ -2,6 +2,7 @@
// names, the island and sync-worker scripts, and the app's pages last. Split from `cmd-dev.ts` at its
// 500-line ceiling; `serve.ts` composes the production table from the same builders.
+import type { RealtimeConfig } from '@ultimat3/core';
import type { Route } from '@ultimat3/http';
import { describeRoutes } from '@ultimat3/render';
import type { Storage } from '@ultimat3/storage';
@@ -19,6 +20,7 @@ import { loadPwaArtifacts } from './pwa-artifacts';
import { assetRoutes } from './runtime-assets';
import { appRoutes } from './runtime-render';
import { servedStorage, storageRoutes } from './runtime-storage';
+import { seoRoutes } from './seo-routes';
import { styleBundle } from './style-bundle';
import { styleRoutes } from './style-routes';
import { serviceWorkerArtifacts } from './sw-artifacts';
@@ -34,6 +36,8 @@ export interface DevRouteTableInput {
readonly dashboard: DevDashboardInput;
/** A getter: the watcher tick rebuilds the islands, and a captured bundle would serve the first. */
readonly islands: () => IslandBundle;
+ /** `app.config.ts`'s `realtime`, as the boot obeyed it: off, no document names a sync node. */
+ readonly realtime: Pick;
}
export interface DevRouteTable {
@@ -52,7 +56,7 @@ export async function devRouteTable(input: DevRouteTableInput): Promise styleBundle()),
+ // `robots.txt` and `sitemap.xml`, the same two files the static export writes (`site-seo.ts`).
+ ...seoRoutes({ env: input.env }),
// `x shot --island`'s harness, in the `/_x` dev namespace so no app route can shadow it. It
// lives here rather than in a second server because everything it needs is in THIS process:
// the built chunks, the app's stylesheet registry, and the one embedded Postgres a checkout
@@ -110,7 +116,7 @@ export async function devRouteTable(input: DevRouteTableInput): Promise input.islands().resolverFor(file),
- sync: sync.head,
+ ...(sync.head === undefined ? {} : { sync: sync.head }),
persisted: sync.persisted,
themeHead: theme.head,
...(pwa === undefined ? {} : { pwaHead: pwa.head + (serviceWorker?.head ?? '') }),
diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts
index cd452ea7a..e730ebdc3 100644
--- a/packages/cli/src/index.ts
+++ b/packages/cli/src/index.ts
@@ -145,6 +145,8 @@ export {
serveApp,
} from './serve';
export { quoteArg } from './shell-quote';
+export type { SiteSeo, SiteSeoOptions } from './site-seo';
+export { ROBOTS_PATH, SITEMAP_PATH, siteSeo } from './site-seo';
export { eachSourceFile, isGenerated, isTest, SOURCE_GLOBS } from './source-files';
export type { SkippedRoute, StaticReport } from './static-report';
export { parseStaticReport } from './static-report';
diff --git a/packages/cli/src/page-sync.test.ts b/packages/cli/src/page-sync.test.ts
index 1cf8cf59c..a6f6c6565 100644
--- a/packages/cli/src/page-sync.test.ts
+++ b/packages/cli/src/page-sync.test.ts
@@ -6,13 +6,15 @@ import { afterAll, expect, test } from 'bun:test';
import { clearRegistry, entity, uuid } from '@ultimat3/entity';
import { pageSync } from './page-sync';
+const ON = { enabled: true } as const;
+
afterAll(() => {
clearRegistry();
});
test('persisted names every entity declared persist: true, and only those, read at call time', async () => {
// OUTSIDE the checkout, so no realtime resolves and no worker is built — this is not about it.
- const sync = await pageSync(Bun.env['TMPDIR'] ?? '/tmp', {}, 'build-under-test');
+ const sync = await pageSync(Bun.env['TMPDIR'] ?? '/tmp', {}, 'build-under-test', ON);
entity('page_sync_kept', { persist: true, columns: { id: uuid().primaryKey() } });
entity('page_sync_dropped', { columns: { id: uuid().primaryKey() } });
@@ -22,9 +24,21 @@ test('persisted names every entity declared persist: true, and only those, read
test('the scripts it serves are the scripts it hands the service worker to precache', async () => {
// The checkout's own realtime resolves from here, so both scripts are built.
- const sync = await pageSync(`${import.meta.dir}/../../realtime`, {}, 'build-under-test');
+ const sync = await pageSync(`${import.meta.dir}/../../realtime`, {}, 'build-under-test', ON);
const urls = sync.scripts.map((script) => script.url);
expect(urls.some((url) => url.startsWith('/_x/page-boot/'))).toBe(true);
expect(urls.some((url) => url.startsWith('/_x/sync-worker/'))).toBe(true);
- expect(sync.head.bootUrl).toBe(urls.find((url) => url.startsWith('/_x/page-boot/')) ?? '');
+ expect(sync.head?.bootUrl).toBe(urls.find((url) => url.startsWith('/_x/page-boot/')) ?? '');
+});
+
+// Reported by an app with `realtime: { enabled: false }`: no sync node is started (`role-realtime.ts`),
+// yet every page still carried `` and the worker and
+// boot scripts — a target nothing serves, for a page runtime to dial.
+test('realtime off: no sync target, no worker, no boot script, nothing to precache', async () => {
+ const sync = await pageSync(`${import.meta.dir}/../../realtime`, {}, 'build-under-test', {
+ enabled: false,
+ });
+ expect(sync.head).toBeUndefined();
+ expect(sync.routes).toEqual([]);
+ expect(sync.scripts).toEqual([]);
});
diff --git a/packages/cli/src/page-sync.ts b/packages/cli/src/page-sync.ts
index f47a5ee64..26f2eff94 100644
--- a/packages/cli/src/page-sync.ts
+++ b/packages/cli/src/page-sync.ts
@@ -3,6 +3,7 @@
// `ultimate-build`, `ultimate-sync-worker`, the boot script), and the persisted record types a
// private document names. One call from both boots, so the two cannot serve different targets.
+import type { RealtimeConfig } from '@ultimat3/core';
import { persistedRecordTypes } from '@ultimat3/entity';
import type { Route } from '@ultimat3/http';
import type { ClientSyncHead } from '@ultimat3/render';
@@ -19,7 +20,11 @@ export interface PageSync {
readonly routes: readonly Route[];
/** The scripts those routes serve, for the service worker to precache beside the islands. */
readonly scripts: readonly FrameworkScript[];
- readonly head: ClientSyncHead;
+ /**
+ * `undefined` when `realtime.enabled` is false: no node is started (`role-realtime.ts`), so a
+ * document naming one would hand the page runtime a target nothing serves.
+ */
+ readonly head: ClientSyncHead | undefined;
/**
* The record types the app persists, read per render off the entity registry — the app's modules
* register their entities during boot, so a value captured here could predate them.
@@ -36,7 +41,13 @@ export async function pageSync(
root: string,
env: Readonly>,
buildId: string,
+ realtime: Pick,
): Promise {
+ // Off: no `ultimate-sync`, no worker, no boot script — the boot is the outbox and the disk
+ // restore, which exist to feed the socket. Checked before either build, which would be wasted.
+ if (!realtime.enabled) {
+ return { routes: [], scripts: [], head: undefined, persisted: persistedRecordTypes };
+ }
const syncUrl = syncUrlFrom(env);
const worker = await buildSyncWorker(root);
const boot = await buildPageBoot(root);
diff --git a/packages/cli/src/seo-routes.test.ts b/packages/cli/src/seo-routes.test.ts
new file mode 100644
index 000000000..da71091cd
--- /dev/null
+++ b/packages/cli/src/seo-routes.test.ts
@@ -0,0 +1,162 @@
+// `robots.txt` and `sitemap.xml` from a RUNNING web role, through `@ultimat3/http`'s real pipeline
+// and beside the app's own pages. Reported by an app: the static export wrote both files and a
+// `ROLE=web` container answered 404 for each — and an app has no way to add a non-page GET route.
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { createServer, defineHttpConfig } from '@ultimat3/http';
+import type { RouteConfig } from '@ultimat3/render';
+import { clearRoutes, defineRoute, registerRoute } from '@ultimat3/render';
+import { appRoutes } from './runtime-render';
+import { seoRoutes } from './seo-routes';
+import { siteSeo } from './site-seo';
+
+const BUILD_ID = 'seo-under-test';
+
+const serve = (
+ env: Readonly>,
+): ReturnType =>
+ createServer({
+ routes: [...seoRoutes({ env }), ...appRoutes({ buildId: BUILD_ID })],
+ role: 'web',
+ config: defineHttpConfig({ dev: true, buildId: BUILD_ID, rateLimit: { scope: 'process' } }),
+ });
+
+const route = (file: string, patch: Partial = {}): void => {
+ registerRoute({
+ file,
+ config: defineRoute({
+ render: 'static',
+ hydrate: 'never',
+ offline: 'network-only',
+ meta: () => ({ title: 'Page', description: 'a page' }),
+ ...patch,
+ } as Parameters[0]),
+ component: () => 'page',
+ });
+};
+
+/** A small site with one of each thing a sitemap has to decide about. */
+const site = (): void => {
+ route('apps/web/site/page.tsx');
+ route('apps/web/site/pricing/page.tsx');
+ route('apps/web/site/blog/[slug]/page.tsx', { prerender: () => ['hello', 'world'] });
+ // `meta` says noindex: the page asks crawlers to stay away, so the sitemap must not invite them.
+ route('apps/web/site/thanks/page.tsx', {
+ meta: async () => ({
+ title: 'Thanks',
+ description: 'after the form',
+ robots: { index: false },
+ }),
+ });
+ // Behind a policy: not public, whatever surface it sits on.
+ route('apps/web/site/members/page.tsx', {
+ render: 'ssr',
+ policy: { permission: 'members:read' },
+ });
+ route('apps/web/app/dashboard/page.tsx', { render: 'ssr' });
+};
+
+const PRODUCTION = { ULTIMATE_ENV: 'production', APP_URL: 'https://www.example.com/' };
+
+beforeEach(() => {
+ clearRoutes();
+});
+
+afterEach(() => {
+ clearRoutes();
+});
+
+describe('siteSeo', () => {
+ test('the sitemap lists every public site page, absolute, and nothing else', async () => {
+ site();
+ const seo = await siteSeo({ baseUrl: 'https://www.example.com', environment: 'production' });
+ const [sitemap] = seo.sitemaps;
+ expect(seo.sitemaps).toHaveLength(1);
+ expect(sitemap?.path).toBe('/sitemap.xml');
+ const locs = [...(sitemap?.xml ?? '').matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1]);
+ expect(locs.sort()).toEqual([
+ 'https://www.example.com',
+ 'https://www.example.com/blog/hello',
+ 'https://www.example.com/blog/world',
+ 'https://www.example.com/pricing',
+ ]);
+ });
+
+ test('robots allows the crawl and names the absolute sitemap in production', async () => {
+ site();
+ const { robots } = await siteSeo({
+ baseUrl: 'https://www.example.com',
+ environment: 'production',
+ });
+ expect(robots).toContain('Allow: /');
+ expect(robots).toContain('Sitemap: https://www.example.com/sitemap.xml');
+ expect(robots).not.toContain('Disallow: /\n');
+ });
+
+ test('anything but production disallows everything and advertises no sitemap', async () => {
+ site();
+ const { robots } = await siteSeo({
+ baseUrl: 'https://www.example.com',
+ environment: 'staging',
+ });
+ expect(robots).toContain('Disallow: /');
+ expect(robots).not.toContain('Sitemap:');
+ });
+
+ // The static export's half: a dynamic route contributes exactly the pages the build EMITTED,
+ // which the caller hands in, rather than a second call to `prerender()`.
+ test('pagesFor replaces prerender() for the dynamic routes, and only for them', async () => {
+ site();
+ const seo = await siteSeo({
+ baseUrl: 'https://www.example.com',
+ environment: 'production',
+ pagesFor: (path) => (path === '/blog/:slug' ? ['/blog/hello'] : []),
+ });
+ const xml = seo.sitemaps[0]?.xml ?? '';
+ expect(xml).toContain('https://www.example.com/blog/hello');
+ expect(xml).not.toContain('/blog/world');
+ expect(xml).toContain('https://www.example.com/pricing');
+ });
+});
+
+describe('seoRoutes, served by the web role', () => {
+ test('GET /robots.txt and GET /sitemap.xml answer, from APP_URL, beside the pages', async () => {
+ site();
+ const server = serve(PRODUCTION);
+
+ const robots = await server.fetch(new Request('http://10.0.0.7:3000/robots.txt'));
+ expect(robots.status).toBe(200);
+ expect(robots.headers.get('content-type')).toContain('text/plain');
+ expect(await robots.text()).toContain('Sitemap: https://www.example.com/sitemap.xml');
+
+ const sitemap = await server.fetch(new Request('http://10.0.0.7:3000/sitemap.xml'));
+ expect(sitemap.status).toBe(200);
+ expect(sitemap.headers.get('content-type')).toContain('application/xml');
+ const xml = await sitemap.text();
+ expect(xml).toContain('https://www.example.com/pricing');
+ expect(xml).not.toContain('10.0.0.7');
+
+ // And the pages are still the pages.
+ expect((await server.fetch(new Request('http://10.0.0.7:3000/pricing'))).status).toBe(200);
+ });
+
+ test('a non-production process serves a robots.txt that refuses the crawl', async () => {
+ site();
+ const server = serve({ ULTIMATE_ENV: 'staging', APP_URL: 'https://staging.example.com' });
+ const body = await (await server.fetch(new Request('http://x/robots.txt'))).text();
+ expect(body).toContain('Disallow: /');
+ expect(body).not.toContain('Sitemap:');
+ });
+
+ test('with no APP_URL the static build`s SITE_ORIGIN is the origin, then the request`s', async () => {
+ site();
+ const fromSite = serve({ ULTIMATE_ENV: 'production', SITE_ORIGIN: 'https://site.example.com' });
+ expect(await (await fromSite.fetch(new Request('http://x/sitemap.xml'))).text()).toContain(
+ 'https://site.example.com',
+ );
+ const fromRequest = serve({ ULTIMATE_ENV: 'production' });
+ expect(
+ await (await fromRequest.fetch(new Request('https://req.example.com/sitemap.xml'))).text(),
+ ).toContain('https://req.example.com');
+ });
+});
diff --git a/packages/cli/src/seo-routes.ts b/packages/cli/src/seo-routes.ts
new file mode 100644
index 000000000..30ad72854
--- /dev/null
+++ b/packages/cli/src/seo-routes.ts
@@ -0,0 +1,67 @@
+// `GET /robots.txt` and `GET /sitemap.xml` from a running web role — the files the static export
+// writes, answered by the process for a deploy that serves its pages from a container. Mounted by
+// `x dev` and `runRole` alike, before the app's pages, for `style-routes.ts`' reason.
+
+import { DEFAULT_ENVIRONMENT, type Environment, tryResolveEnvironment } from '@ultimat3/core';
+import type { Route, UltimateRequest } from '@ultimat3/http';
+import { applyCacheHeaders } from '@ultimat3/http';
+import { ROBOTS_PATH, SITEMAP_PATH, siteSeo } from './site-seo';
+
+export interface SeoRoutesOptions {
+ readonly env: Readonly>;
+}
+
+/**
+ * The public origin: `APP_URL`, the one the framework's runtime already names for it (OAuth's
+ * redirect, the sync node's admitted origin); else `SITE_ORIGIN`, the static build's; else the
+ * request's own. A container behind an ingress sees its pod address as the request's host, which
+ * is why the declared origin comes first — a sitemap of `http://10.0.0.7:3000/…` indexes nothing.
+ */
+function originOf(env: SeoRoutesOptions['env'], request: UltimateRequest): string {
+ for (const key of ['APP_URL', 'SITE_ORIGIN']) {
+ const declared = env[key]?.trim() ?? '';
+ if (declared !== '') return declared.replace(/\/+$/, '');
+ }
+ return new URL(request.url).origin;
+}
+
+/**
+ * Per request, never cached across one: `prerender()` may enumerate rows that change, and a
+ * crawler asks for these a handful of times a day. An hour of shared cache is what a CDN keeps.
+ */
+const SEO_CACHE = { mode: 'public', maxAgeSeconds: 3600 } as const;
+
+export function seoRoutes(options: SeoRoutesOptions): readonly Route[] {
+ const environment: Environment =
+ tryResolveEnvironment({ env: options.env }) ?? DEFAULT_ENVIRONMENT;
+ const answer = async (request: UltimateRequest) =>
+ await siteSeo({ baseUrl: originOf(options.env, request), environment });
+
+ return [
+ {
+ method: 'GET',
+ path: ROBOTS_PATH,
+ meta: { name: 'seo.robots', auth: 'public', tags: ['seo'] },
+ handler: async (request: UltimateRequest): Promise =>
+ applyCacheHeaders(
+ new Response((await answer(request)).robots, {
+ headers: { 'content-type': 'text/plain; charset=utf-8' },
+ }),
+ SEO_CACHE,
+ ),
+ },
+ {
+ method: 'GET',
+ path: SITEMAP_PATH,
+ meta: { name: 'seo.sitemap', auth: 'public', tags: ['seo'] },
+ // The first file is `/sitemap.xml` in both shapes: the whole urlset, or the index of parts.
+ handler: async (request: UltimateRequest): Promise =>
+ applyCacheHeaders(
+ new Response((await answer(request)).sitemaps[0]?.xml ?? '', {
+ headers: { 'content-type': 'application/xml; charset=utf-8' },
+ }),
+ SEO_CACHE,
+ ),
+ },
+ ];
+}
diff --git a/packages/cli/src/serve-boot.ts b/packages/cli/src/serve-boot.ts
index 2c92167da..37b13fc61 100644
--- a/packages/cli/src/serve-boot.ts
+++ b/packages/cli/src/serve-boot.ts
@@ -25,6 +25,7 @@ import { appRoutes } from './runtime-render';
import { replicaOverrides } from './runtime-replica';
import type { RunningServices } from './runtime-services';
import { servedStorage, storageRoutes } from './runtime-storage';
+import { seoRoutes } from './seo-routes';
import { loadDrainConfig } from './serve-drain';
import { configureReporting, containerBinding, metricsPortFor, portFromEnv } from './serve-env';
import type { ServedApp, ServeOptions } from './serve-types';
@@ -139,7 +140,7 @@ async function webSurface(
const theme = themeBoot(await loadThemeMode(options.root));
// The page's sync target and its scripts — the same call `x dev` makes, so the two cannot differ.
// Before the service worker, which precaches those scripts.
- const sync = await pageSync(options.root, options.env, buildId);
+ const sync = await pageSync(options.root, options.env, buildId, runtime.realtime);
// The worker, from the SAME route table this process is about to serve — `describeRoutes()` is
// the one projection `x.manifest.json`, `/_x`, the sitemap and `sw.js` are all built from, so a
// route added here cannot be missing from the precache manifest.
@@ -173,12 +174,14 @@ async function webSurface(
// The surface stylesheets the documents link. Built from the registry the `loadApp` above
// filled, so this process serves exactly the CSS it renders against.
...styleRoutes(() => styleBundle()),
+ // `robots.txt` and `sitemap.xml`, the same two files the static export writes (`site-seo.ts`).
+ ...seoRoutes({ env: options.env }),
// The page's one socket: its worker script, served beside the islands for their reason.
...sync.routes,
...appRoutes({
buildId,
resolveIsland: (file) => islands.resolverFor(file),
- sync: sync.head,
+ ...(sync.head === undefined ? {} : { sync: sync.head }),
persisted: sync.persisted,
themeHead: theme.head,
...(pwa === undefined ? {} : { pwaHead: pwa.head + (serviceWorker?.head ?? '') }),
diff --git a/packages/cli/src/site-seo.ts b/packages/cli/src/site-seo.ts
new file mode 100644
index 000000000..09aa1f540
--- /dev/null
+++ b/packages/cli/src/site-seo.ts
@@ -0,0 +1,78 @@
+// `robots.txt` and `sitemap.xml`, off the route table: ONE answer for the static export
+// (`apps/web/prerender.ts` writes it into the artifact) and the running web role (`seo-routes.ts`
+// serves it), so a crawler reads the same two files from a CDN and from a container.
+
+import type { Environment } from '@ultimat3/core';
+import type { RouteEntry } from '@ultimat3/render';
+import { routeEntries } from '@ultimat3/render';
+import { enumeratePrerender, fillPath } from '@ultimat3/render/server';
+import type { RouteRecord, SitemapFile } from '@ultimat3/seo';
+import { buildRobots, buildSitemap, isDynamic } from '@ultimat3/seo';
+import { readSiteMeta } from './seo-meta';
+
+export const ROBOTS_PATH = '/robots.txt';
+export const SITEMAP_PATH = '/sitemap.xml';
+
+export interface SiteSeoOptions {
+ /** The public origin every `` and the `Sitemap:` line are absolute against. */
+ readonly baseUrl: string;
+ /** Omitted: `ULTIMATE_ENV`, read by `@ultimat3/seo` — anything but `production` disallows. */
+ readonly environment?: Environment | undefined;
+ /**
+ * The concrete pages a build EMITTED for a dynamic route. The static export passes its report's,
+ * so the sitemap cannot name a page the artifact lacks; absent, `prerender()` is asked — which is
+ * the same list, enumerated by the same function the prerenderer calls.
+ */
+ readonly pagesFor?: ((routePath: string) => readonly string[]) | undefined;
+}
+
+export interface SiteSeo {
+ readonly robots: string;
+ /** Every sitemap file: `/sitemap.xml` alone, or the index at that path first and its parts after. */
+ readonly sitemaps: readonly SitemapFile[];
+}
+
+/** A `site/` page anyone may fetch. A policy on a `site/` route makes it not public, whatever the surface. */
+const isPublicSite = (entry: RouteEntry): boolean =>
+ entry.surface === 'site' && entry.config.policy === undefined;
+
+const pagesOf = async (entry: RouteEntry, options: SiteSeoOptions): Promise => {
+ if (options.pagesFor !== undefined) return options.pagesFor(entry.path);
+ const params = await enumeratePrerender(entry);
+ return params.map((set) => fillPath(entry.pattern.source, set));
+};
+
+/**
+ * The public `site/` routes as `@ultimat3/seo` reads them. `meta` is carried where it resolves
+ * without a request (`readSiteMeta`), which is what lets a page's own `robots: { index: false }`
+ * keep it out of the sitemap — a page that asks crawlers to stay away must not be listed for them.
+ */
+async function publicSiteRoutes(options: SiteSeoOptions): Promise {
+ const metaByPath = new Map((await readSiteMeta()).records.map((r) => [r.path, r.meta]));
+ return routeEntries()
+ .filter(isPublicSite)
+ .map((entry) => {
+ const meta = metaByPath.get(entry.path);
+ return {
+ path: entry.path,
+ file: entry.file,
+ surface: 'site' as const,
+ render: entry.config.render,
+ ...(meta === undefined ? {} : { meta }),
+ ...(isDynamic(entry.path) ? { prerender: () => pagesOf(entry, options) } : {}),
+ };
+ });
+}
+
+export async function siteSeo(options: SiteSeoOptions): Promise {
+ const sitemap = await buildSitemap(await publicSiteRoutes(options), { baseUrl: options.baseUrl });
+ // Past 50,000 URLs `files` are `/sitemap-N.xml` and the index is `/sitemap.xml`; below it,
+ // `files` is that one file and there is no index.
+ const sitemaps = sitemap.index === undefined ? sitemap.files : [sitemap.index, ...sitemap.files];
+ const robots = buildRobots({
+ baseUrl: options.baseUrl,
+ sitemaps: [SITEMAP_PATH],
+ ...(options.environment === undefined ? {} : { environment: options.environment }),
+ });
+ return { robots, sitemaps };
+}
diff --git a/packages/cli/src/templates/scaffold-entries.ts b/packages/cli/src/templates/scaffold-entries.ts
index 94e90711e..663ad8f20 100644
--- a/packages/cli/src/templates/scaffold-entries.ts
+++ b/packages/cli/src/templates/scaffold-entries.ts
@@ -65,9 +65,7 @@ const prerender =
// landed on disk — which is what \`x build --target static --json\` reads back.
import { join } from 'node:path';
-import { DEFAULT_ORIGIN, type PrerenderReport, prerenderSite } from '@ultimat3/cli';
-import { routeEntries } from '@ultimat3/render';
-import { buildRobots, buildSitemap, type RouteRecord } from '@ultimat3/seo';
+import { DEFAULT_ORIGIN, type PrerenderReport, prerenderSite, siteSeo } from '@ultimat3/cli';
const root = join(import.meta.dir, '..', '..');
const flag = Bun.argv.indexOf('--out');
@@ -79,42 +77,28 @@ const out = (flag === -1 ? undefined : Bun.argv[flag + 1]) ?? join(root, '.x', '
const origin = Bun.env.SITE_ORIGIN;
/**
- * The route table as \`@ultimat3/seo\` reads it. \`RouteRecord\` is a static row and \`defineRoute\`
- * is a live declaration, so one has to be projected onto the other — and \`prerenderSite\` has
- * already loaded the app by the time this runs, which is what fills \`routeEntries()\`.
+ * \`sitemap.xml\` and \`robots.txt\`, into the same directory the HTML went: a static export is
+ * served with no process behind it, so the CDN needs the files. \`siteSeo\` is the same answer a
+ * running web role serves at \`/sitemap.xml\` and \`/robots.txt\` — the public \`site/\` routes, a
+ * page whose \`meta\` says \`robots: { index: false }\` left out — so a crawler reads one sitemap
+ * from the CDN and from the container.
*
- * A DYNAMIC route contributes exactly the URLs this build enumerated for it, read back off the
- * report rather than by calling \`prerender()\` a second time: the sitemap then cannot name a page
- * the artifact does not contain, which is the only failure mode a sitemap really has.
- */
-const siteRoutes = (report: PrerenderReport): readonly RouteRecord[] =>
- routeEntries()
- .filter((entry) => entry.surface === 'site')
- .map((entry) => ({
- path: entry.path,
- file: entry.file,
- surface: 'site' as const,
- render: entry.config.render,
- prerender: () =>
- report.pages.filter((page) => page.route === entry.path).map((page) => page.path),
- }));
-
-/**
- * \`sitemap.xml\` and \`robots.txt\`, into the same directory the HTML went. Both belong to the
- * ARTIFACT rather than to a request: a static export is served with no process behind it, so a
- * route that answered them at run time would be a file the CDN never has.
+ * A DYNAMIC route contributes exactly the URLs this build emitted for it, read back off the report
+ * rather than by calling \`prerender()\` a second time: the sitemap then cannot name a page the
+ * artifact does not contain.
*
- * \`buildRobots\` fails closed — anything that is not \`ULTIMATE_ENV=production\` emits
+ * \`robots.txt\` fails closed — anything that is not \`ULTIMATE_ENV=production\` emits
* \`Disallow: /\` and advertises no sitemap — so a preview build cannot outrank the real site.
*/
async function writeSeoFiles(report: PrerenderReport, baseUrl: string): Promise {
- const sitemap = await buildSitemap(siteRoutes(report), { baseUrl });
- // Past 50,000 URLs \`files\` are \`/sitemap-N.xml\` and the index is \`/sitemap.xml\`; below it,
- // \`files\` is that one file and there is no index. Writing both lists covers each case once.
- const written = sitemap.index === undefined ? sitemap.files : [sitemap.index, ...sitemap.files];
- for (const file of written) await Bun.write(join(out, file.path), file.xml);
- await Bun.write(join(out, 'robots.txt'), buildRobots({ baseUrl, sitemaps: ['/sitemap.xml'] }));
- return [...written.map((file) => file.path), '/robots.txt'];
+ const seo = await siteSeo({
+ baseUrl,
+ pagesFor: (route) =>
+ report.pages.filter((page) => page.route === route).map((page) => page.path),
+ });
+ for (const file of seo.sitemaps) await Bun.write(join(out, file.path), file.xml);
+ await Bun.write(join(out, 'robots.txt'), seo.robots);
+ return [...seo.sitemaps.map((file) => file.path), '/robots.txt'];
}
if (import.meta.main) {
diff --git a/wiki/Realtime.md b/wiki/Realtime.md
index f0722a269..4e8ebbe5b 100644
--- a/wiki/Realtime.md
+++ b/wiki/Realtime.md
@@ -507,7 +507,7 @@ A client on build `A` connecting to a `sync` node on build `B` is **accepted**,
| `X_CURSOR_STALE` | resume cursor cannot be honoured and no snapshot path was supplied | `pass 'snapshot' to resumeFrom() so the fallback path can re-snapshot instead of failing` |
| `X_REBASE_CONFLICT` | a `custom(merge)` returned something other than a row with a string `id` | `set conflict: 'server-wins' on the mutator, or return a row from custom(merge)` |
| `X_REALTIME_UNINSTALLED` | a hook ran in a browser island whose bundle never called `installRealtime()` | `x build`, which installs it on every island whose own graph imports realtime. An island that reaches realtime only through a package calls `installRealtime({ signal: createSignal })` in `mount` |
-| `X_SYNC_UNCONFIGURED` | a live hook needed the page socket, and neither `installRealtime({ sync })` nor `` named a target | serve the page through `x dev` or the container, which render the meta, or pass `installRealtime({ signal, sync: { url, buildId } })` |
+| `X_SYNC_UNCONFIGURED` | a live hook needed the page socket, and neither `installRealtime({ sync })` nor `` named a target. The meta is not rendered at all when `app.config.ts` says `realtime: { enabled: false }`: no node is started, so there is nothing to dial (22.2.2) | serve the page through `x dev` or the container, which render the meta, or pass `installRealtime({ signal, sync: { url, buildId } })` |
| `X_RECORD_REJECTED` | a row reached the record store with no key, or not as an object | `x entities describe --json`, then return whole rows |
| `X_MUTATOR_CLOCK_MISSING` | `conflict: 'last-write-wins'` with no number `updatedAt` on the entity | add the clock, or declare `conflict: 'server-wins'` |
| `X_TRANSPORT_UNAVAILABLE` | the fanout bus is down | `x doctor — then check NATS_URL points at a reachable nats-server` |
diff --git a/wiki/SEO.md b/wiki/SEO.md
index adbc8c20d..31bd7aaf5 100644
--- a/wiki/SEO.md
+++ b/wiki/SEO.md
@@ -48,3 +48,17 @@ budgets are **not** here: they are the `budgets` step and `X_BUDGET_EXCEEDED`.
| `parseImageQuery()` | the one reader of the `?w=&f=&q=` a responsive image URL carries (`X_IMAGE_QUERY_INVALID`) |
Every `X_SEO_*` and image code is in [Error codes](Error-Codes).
+
+## `robots.txt` and `sitemap.xml`
+
+One answer, served two ways. `siteSeo()` from `@ultimat3/cli` builds both from the route table. It
+takes the public `site/` routes (no `policy`), leaves out any page whose `meta` says
+`robots: { index: false }`, and expands a dynamic route through its `prerender()`.
+
+| Where | How |
+|---|---|
+| static export | the scaffolded `apps/web/prerender.ts` writes `siteSeo()`'s files into `.x/static`. A dynamic route lists exactly the pages the build emitted |
+| web role (`x dev`, `runRole`) | `GET /robots.txt` and `GET /sitemap.xml`, `public, max-age=3600`, built per request. Absolute against `APP_URL`, else `SITE_ORIGIN`, else the request's own origin. `As of 2026-09-25` (22.2.2) |
+
+Past 50,000 URLs, `/sitemap.xml` is the index. The web role serves the index but not the
+`/sitemap-N.xml` parts, so a site that large serves its sitemap from the static export.