Share portable catalog sync core

.gitignore

@@ -21,7 +21,6 @@ data/openapi-bundles/
 .entire/
 .eval-logs/
 .pi/
-.repos/effect
 .zed/
 fern/
 docs/my-notes.md

README.md

@@ -41,10 +41,16 @@ The package manager for consumers is npm. Bun remains the runtime because local
 bun run sync -- --db data/catalog.sqlite
 ```
 
-`sync` skips a catalog completed within the last 24 hours. Use `--force` to refresh immediately, or `--max-age-hours <hours>` to choose a different threshold. For a bounded smoke test, use `--limit 1 --force` with a temporary `--db`; partial runs never mark the main catalog fresh or prune unseen packages.
+Ordinary catalog commands automatically synchronize when the local catalog is missing or older than 24 hours. Set `HK_OPEN_DATA_AUTO_SYNC=0` for offline or fully manual operation. The explicit `sync` command supports `--force` to refresh immediately and `--max-age-hours <hours>` to choose a different threshold. For a bounded smoke test, use `--limit 1 --force` with a temporary `--db`; partial runs never mark the main catalog fresh or prune unseen packages.
 
 The sync engine is storage-neutral. The public repository supplies the local SQLite store; a private Cloudflare service can provide a D1 store and invoke the same Effect program from a Cron trigger.
 
+Cloudflare and other non-Bun runtimes should import portable contracts and helpers without loading the SQLite adapter:
+
+```ts
+import { Catalog, CatalogSyncStore, syncCatalog } from "hk-open-data/portable";
+```
+
 ## Use the CLI
 
 ```sh

package.json

@@ -7,7 +7,6 @@
   ],
   "type": "module",
   "scripts": {
-    "prepare": "./scripts/prepare-effect.sh",
     "typecheck": "bun run --filter '*' typecheck",
     "lint": "oxlint .",
     "format": "oxfmt --write .",

packages/cli/src/index.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { defineCommand, runMain } from "citty";
-import { Effect } from "effect";
+import { Effect, Logger } from "effect";
 
 import {
   makeCkanClientLayer,
@@ -14,6 +14,7 @@ import {
 } from "@hk-open-data/core";
 import type { CatalogError, CatalogService } from "@hk-open-data/core";
 import { formatCatalogError } from "./output";
+import { DEFAULT_CATALOG_MAX_AGE_MS, isAutomaticCatalogSyncEnabled } from "./sync-policy";
 
 declare const __HK_OPEN_DATA_VERSION__: string;
 
@@ -415,6 +416,22 @@ async function withCatalog(
   dbPath: string | undefined,
   operation: (catalog: CatalogService) => Effect.Effect<unknown, CatalogError>,
 ): Promise<void> {
+  if (
+    isAutomaticCatalogSyncEnabled() &&
+    !(await runSync(
+      {
+        apiBaseUrl: "https://data.gov.hk/en-data/api/3/action",
+        concurrency: 8,
+        dbPath,
+        force: false,
+        limit: 0,
+        maxAgeMs: DEFAULT_CATALOG_MAX_AGE_MS,
+      },
+      false,
+    ))
+  ) {
+    return;
+  }
   const outcome = await Effect.runPromise(
     Effect.match(
       Effect.scoped(Effect.flatMap(makeSqliteCatalog(dbPath ? { dbPath } : {}), operation)),
@@ -432,15 +449,18 @@ async function withCatalog(
   process.stdout.write(`${JSON.stringify(outcome.data, null, 2)}\n`);
 }
 
-async function runSync(options: {
-  readonly apiBaseUrl: string;
-  readonly concurrency: number;
-  readonly dbPath: string | undefined;
-  readonly force: boolean;
-  readonly limit: number;
-  readonly maxAgeMs: number;
-}): Promise<void> {
-  const program = syncCatalog({
+async function runSync(
+  options: {
+    readonly apiBaseUrl: string;
+    readonly concurrency: number;
+    readonly dbPath: string | undefined;
+    readonly force: boolean;
+    readonly limit: number;
+    readonly maxAgeMs: number;
+  },
+  emitReport = true,
+): Promise<boolean> {
+  let program = syncCatalog({
     apiBaseUrl: options.apiBaseUrl,
     concurrency: options.concurrency,
     force: options.force,
@@ -452,6 +472,7 @@ async function runSync(options: {
       makeSqliteCatalogSyncStoreLayer(options.dbPath ? { dbPath: options.dbPath } : {}),
     ),
   );
+  if (!emitReport) program = program.pipe(Effect.provide(Logger.layer([])));
   const outcome = await Effect.runPromise(
     Effect.scoped(
       Effect.match(program, {
@@ -463,9 +484,10 @@ async function runSync(options: {
   if ("error" in outcome) {
     process.stderr.write(`${JSON.stringify(outcome.error)}\n`);
     process.exitCode = 1;
-    return;
+    return false;
   }
-  process.stdout.write(`${JSON.stringify(outcome.data, null, 2)}\n`);
+  if (emitReport) process.stdout.write(`${JSON.stringify(outcome.data, null, 2)}\n`);
+  return true;
 }
 
 function numberArg(flag: string, value: string | undefined): number {

packages/cli/src/sync-policy.ts

@@ -0,0 +1,8 @@
+export const DEFAULT_CATALOG_MAX_AGE_MS = 24 * 60 * 60 * 1_000;
+
+export function isAutomaticCatalogSyncEnabled(
+  environment: Readonly<Record<string, string | undefined>> = process.env,
+): boolean {
+  const value = environment.HK_OPEN_DATA_AUTO_SYNC?.trim().toLowerCase();
+  return value !== "0" && value !== "false" && value !== "no" && value !== "off";
+}

packages/cli/test/sync-policy.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, test } from "bun:test";
+
+import { DEFAULT_CATALOG_MAX_AGE_MS, isAutomaticCatalogSyncEnabled } from "../src/sync-policy";
+
+describe("automatic catalog synchronization policy", () => {
+  test("defaults to a 24-hour freshness window", () => {
+    expect(DEFAULT_CATALOG_MAX_AGE_MS).toBe(86_400_000);
+    expect(isAutomaticCatalogSyncEnabled({})).toBe(true);
+  });
+
+  test("accepts common opt-out values", () => {
+    for (const value of ["0", "false", "NO", " off "]) {
+      expect(isAutomaticCatalogSyncEnabled({ HK_OPEN_DATA_AUTO_SYNC: value })).toBe(false);
+    }
+  });
+});

packages/core/README.md

@@ -12,6 +12,8 @@ Current scope:
 
 This package stays free of service transports and agent-specific behavior. CLIs and private services import catalog behavior from here and only handle boundary-specific translation.
 
+The `portable` entry exports Effect contracts, CKAN clients, resource retrieval helpers, and normalized synchronization records without importing `bun:sqlite`. Worker and D1 adapters should use it instead of the Bun-oriented root entry.
+
 ## Effect Boundaries
 
 Core operations return `Effect<Success, TypedError>` rather than throwing or returning a bare promise. The main services are:

packages/core/package.json

@@ -5,6 +5,10 @@
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts",
+    "./portable": "./src/portable.ts"
+  },
   "scripts": {
     "typecheck": "tsc --noEmit",
     "get-package": "bun run scripts/get-package.ts",

packages/core/src/catalog/search-index-schema.ts

@@ -0,0 +1,2 @@
+export const DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION = "2";
+export const METADATA_SEARCH_INDEX_SCHEMA_VERSION = "metadata-1";

packages/core/src/catalog/sqlite-search.ts

@@ -4,6 +4,10 @@ import type { Database } from "bun:sqlite";
 
 import { documentKind, type SearchEvidence } from "./catalog-mappers";
 import { InvalidSearchCursor } from "./errors";
+import {
+  DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION,
+  METADATA_SEARCH_INDEX_SCHEMA_VERSION,
+} from "./search-index-schema";
 import type { SearchPackagesInput, SearchResourcesInput } from "./schemas";
 import type { PackageRow, ResourceRow } from "./sqlite-rows";
 import type {
@@ -91,7 +95,6 @@ export interface SqliteSearchDependencies {
 }
 
 const SEARCH_SNIPPET_CHARS = 240;
-const SEARCH_INDEX_SCHEMA_VERSION = "2";
 const RRF_K = 60;
 
 const enum SearchTier {
@@ -112,9 +115,7 @@ export class SqliteCatalogSearch {
     if (
       !this.hasTable("package_search") ||
       !this.hasTable("resource_search") ||
-      !this.hasTable("document_chunk_search") ||
       !this.hasTable("search_exact_names") ||
-      !this.hasTable("search_document_reference_counts") ||
       !this.hasTable("search_index_metadata")
     ) {
       return false;
@@ -124,7 +125,12 @@ export class SqliteCatalogSearch {
         "SELECT value FROM search_index_metadata WHERE key = ? LIMIT 1",
       )
       .get("schema_version");
-    return row?.value === SEARCH_INDEX_SCHEMA_VERSION;
+    if (row?.value === METADATA_SEARCH_INDEX_SCHEMA_VERSION) return true;
+    return (
+      row?.value === DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION &&
+      this.hasTable("document_chunk_search") &&
+      this.hasTable("search_document_reference_counts")
+    );
   }
 
   searchPackages(input: SearchPackagesInput): SearchPackagesOutput {
@@ -596,7 +602,21 @@ export class SqliteCatalogSearch {
   }
 
   private hasDocumentSearchRelations(): boolean {
-    return this.hasTable("spec_document_refs") && this.hasTable("spec_documents");
+    return (
+      this.searchIndexSchemaVersion() === DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION &&
+      this.hasTable("spec_document_refs") &&
+      this.hasTable("spec_documents") &&
+      this.hasTable("document_chunk_search") &&
+      this.hasTable("search_document_reference_counts")
+    );
+  }
+
+  private searchIndexSchemaVersion(): string | undefined {
+    return this.db
+      .query<{ value: string }, []>(
+        "SELECT value FROM search_index_metadata WHERE key = 'schema_version' LIMIT 1",
+      )
+      .get()?.value;
   }
 
   private hasTable(name: string): boolean {

packages/core/src/portable.ts

@@ -0,0 +1,18 @@
+export * from "./ckan/client";
+export * from "./ckan/errors";
+export * from "./ckan/schemas";
+export * from "./ckan/types";
+export * from "./catalog/errors";
+export * from "./catalog/catalog-mappers";
+export * from "./catalog/platform-api";
+export * from "./catalog/resource-fetch";
+export * from "./catalog/search-index-schema";
+export * from "./catalog/schemas";
+export * from "./catalog/service";
+export * from "./catalog/sqlite-rows";
+export * from "./catalog/types";
+export * from "./sync/errors";
+export * from "./sync/records";
+export * from "./sync/schema";
+export * from "./sync/service";
+export * from "./sync/types";