From 5f9cda0423eaf1b34764371a3441aaf0e6f34df0 Mon Sep 17 00:00:00 2001 From: jj Date: Sat, 8 Aug 2026 03:28:50 +0800 Subject: [PATCH] feat: share portable catalog sync core --- .gitignore | 1 - README.md | 8 +- package.json | 1 - packages/cli/src/index.ts | 46 +++-- packages/cli/src/sync-policy.ts | 8 + packages/cli/test/sync-policy.test.ts | 16 ++ packages/core/README.md | 2 + packages/core/package.json | 4 + .../core/src/catalog/search-index-schema.ts | 2 + packages/core/src/catalog/sqlite-search.ts | 30 +++- packages/core/src/portable.ts | 18 ++ packages/core/src/sync/records.ts | 160 ++++++++++++++++++ packages/core/src/sync/sqlite.ts | 143 ++++++---------- packages/core/test/sync/sqlite.test.ts | 20 +++ packages/hk-open-data/README.md | 4 + packages/hk-open-data/package.json | 4 + packages/hk-open-data/scripts/build.ts | 1 + packages/hk-open-data/src/portable.ts | 1 + scripts/README.md | 4 +- scripts/prepare-effect.sh | 13 -- 20 files changed, 362 insertions(+), 124 deletions(-) create mode 100644 packages/cli/src/sync-policy.ts create mode 100644 packages/cli/test/sync-policy.test.ts create mode 100644 packages/core/src/catalog/search-index-schema.ts create mode 100644 packages/core/src/portable.ts create mode 100644 packages/core/src/sync/records.ts create mode 100644 packages/hk-open-data/src/portable.ts delete mode 100755 scripts/prepare-effect.sh diff --git a/.gitignore b/.gitignore index 437360c..759a677 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ data/openapi-bundles/ .entire/ .eval-logs/ .pi/ -.repos/effect .zed/ fern/ docs/my-notes.md diff --git a/README.md b/README.md index 28d8add..d2ef8d6 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,16 @@ The package manager for consumers is npm. Bun remains the runtime because local bun run sync -- --db data/catalog.sqlite ``` -`sync` skips a catalog completed within the last 24 hours. Use `--force` to refresh immediately, or `--max-age-hours ` to choose a different threshold. For a bounded smoke test, use `--limit 1 --force` with a temporary `--db`; partial runs never mark the main catalog fresh or prune unseen packages. +Ordinary catalog commands automatically synchronize when the local catalog is missing or older than 24 hours. Set `HK_OPEN_DATA_AUTO_SYNC=0` for offline or fully manual operation. The explicit `sync` command supports `--force` to refresh immediately and `--max-age-hours ` to choose a different threshold. For a bounded smoke test, use `--limit 1 --force` with a temporary `--db`; partial runs never mark the main catalog fresh or prune unseen packages. The sync engine is storage-neutral. The public repository supplies the local SQLite store; a private Cloudflare service can provide a D1 store and invoke the same Effect program from a Cron trigger. +Cloudflare and other non-Bun runtimes should import portable contracts and helpers without loading the SQLite adapter: + +```ts +import { Catalog, CatalogSyncStore, syncCatalog } from "hk-open-data/portable"; +``` + ## Use the CLI ```sh diff --git a/package.json b/package.json index f5ed1af..94b5b03 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,6 @@ ], "type": "module", "scripts": { - "prepare": "./scripts/prepare-effect.sh", "typecheck": "bun run --filter '*' typecheck", "lint": "oxlint .", "format": "oxfmt --write .", diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index d0abcb2..2939519 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun import { defineCommand, runMain } from "citty"; -import { Effect } from "effect"; +import { Effect, Logger } from "effect"; import { makeCkanClientLayer, @@ -14,6 +14,7 @@ import { } from "@hk-open-data/core"; import type { CatalogError, CatalogService } from "@hk-open-data/core"; import { formatCatalogError } from "./output"; +import { DEFAULT_CATALOG_MAX_AGE_MS, isAutomaticCatalogSyncEnabled } from "./sync-policy"; declare const __HK_OPEN_DATA_VERSION__: string; @@ -415,6 +416,22 @@ async function withCatalog( dbPath: string | undefined, operation: (catalog: CatalogService) => Effect.Effect, ): Promise { + if ( + isAutomaticCatalogSyncEnabled() && + !(await runSync( + { + apiBaseUrl: "https://data.gov.hk/en-data/api/3/action", + concurrency: 8, + dbPath, + force: false, + limit: 0, + maxAgeMs: DEFAULT_CATALOG_MAX_AGE_MS, + }, + false, + )) + ) { + return; + } const outcome = await Effect.runPromise( Effect.match( Effect.scoped(Effect.flatMap(makeSqliteCatalog(dbPath ? { dbPath } : {}), operation)), @@ -432,15 +449,18 @@ async function withCatalog( process.stdout.write(`${JSON.stringify(outcome.data, null, 2)}\n`); } -async function runSync(options: { - readonly apiBaseUrl: string; - readonly concurrency: number; - readonly dbPath: string | undefined; - readonly force: boolean; - readonly limit: number; - readonly maxAgeMs: number; -}): Promise { - const program = syncCatalog({ +async function runSync( + options: { + readonly apiBaseUrl: string; + readonly concurrency: number; + readonly dbPath: string | undefined; + readonly force: boolean; + readonly limit: number; + readonly maxAgeMs: number; + }, + emitReport = true, +): Promise { + let program = syncCatalog({ apiBaseUrl: options.apiBaseUrl, concurrency: options.concurrency, force: options.force, @@ -452,6 +472,7 @@ async function runSync(options: { makeSqliteCatalogSyncStoreLayer(options.dbPath ? { dbPath: options.dbPath } : {}), ), ); + if (!emitReport) program = program.pipe(Effect.provide(Logger.layer([]))); const outcome = await Effect.runPromise( Effect.scoped( Effect.match(program, { @@ -463,9 +484,10 @@ async function runSync(options: { if ("error" in outcome) { process.stderr.write(`${JSON.stringify(outcome.error)}\n`); process.exitCode = 1; - return; + return false; } - process.stdout.write(`${JSON.stringify(outcome.data, null, 2)}\n`); + if (emitReport) process.stdout.write(`${JSON.stringify(outcome.data, null, 2)}\n`); + return true; } function numberArg(flag: string, value: string | undefined): number { diff --git a/packages/cli/src/sync-policy.ts b/packages/cli/src/sync-policy.ts new file mode 100644 index 0000000..b3749b0 --- /dev/null +++ b/packages/cli/src/sync-policy.ts @@ -0,0 +1,8 @@ +export const DEFAULT_CATALOG_MAX_AGE_MS = 24 * 60 * 60 * 1_000; + +export function isAutomaticCatalogSyncEnabled( + environment: Readonly> = process.env, +): boolean { + const value = environment.HK_OPEN_DATA_AUTO_SYNC?.trim().toLowerCase(); + return value !== "0" && value !== "false" && value !== "no" && value !== "off"; +} diff --git a/packages/cli/test/sync-policy.test.ts b/packages/cli/test/sync-policy.test.ts new file mode 100644 index 0000000..e6ff050 --- /dev/null +++ b/packages/cli/test/sync-policy.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from "bun:test"; + +import { DEFAULT_CATALOG_MAX_AGE_MS, isAutomaticCatalogSyncEnabled } from "../src/sync-policy"; + +describe("automatic catalog synchronization policy", () => { + test("defaults to a 24-hour freshness window", () => { + expect(DEFAULT_CATALOG_MAX_AGE_MS).toBe(86_400_000); + expect(isAutomaticCatalogSyncEnabled({})).toBe(true); + }); + + test("accepts common opt-out values", () => { + for (const value of ["0", "false", "NO", " off "]) { + expect(isAutomaticCatalogSyncEnabled({ HK_OPEN_DATA_AUTO_SYNC: value })).toBe(false); + } + }); +}); diff --git a/packages/core/README.md b/packages/core/README.md index 27ae5a9..403f338 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -12,6 +12,8 @@ Current scope: This package stays free of service transports and agent-specific behavior. CLIs and private services import catalog behavior from here and only handle boundary-specific translation. +The `portable` entry exports Effect contracts, CKAN clients, resource retrieval helpers, and normalized synchronization records without importing `bun:sqlite`. Worker and D1 adapters should use it instead of the Bun-oriented root entry. + ## Effect Boundaries Core operations return `Effect` rather than throwing or returning a bare promise. The main services are: diff --git a/packages/core/package.json b/packages/core/package.json index 77e7852..272bc8d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -5,6 +5,10 @@ "type": "module", "main": "./src/index.ts", "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./portable": "./src/portable.ts" + }, "scripts": { "typecheck": "tsc --noEmit", "get-package": "bun run scripts/get-package.ts", diff --git a/packages/core/src/catalog/search-index-schema.ts b/packages/core/src/catalog/search-index-schema.ts new file mode 100644 index 0000000..9dff919 --- /dev/null +++ b/packages/core/src/catalog/search-index-schema.ts @@ -0,0 +1,2 @@ +export const DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION = "2"; +export const METADATA_SEARCH_INDEX_SCHEMA_VERSION = "metadata-1"; diff --git a/packages/core/src/catalog/sqlite-search.ts b/packages/core/src/catalog/sqlite-search.ts index 5d9c16f..fa44052 100644 --- a/packages/core/src/catalog/sqlite-search.ts +++ b/packages/core/src/catalog/sqlite-search.ts @@ -4,6 +4,10 @@ import type { Database } from "bun:sqlite"; import { documentKind, type SearchEvidence } from "./catalog-mappers"; import { InvalidSearchCursor } from "./errors"; +import { + DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION, + METADATA_SEARCH_INDEX_SCHEMA_VERSION, +} from "./search-index-schema"; import type { SearchPackagesInput, SearchResourcesInput } from "./schemas"; import type { PackageRow, ResourceRow } from "./sqlite-rows"; import type { @@ -91,7 +95,6 @@ export interface SqliteSearchDependencies { } const SEARCH_SNIPPET_CHARS = 240; -const SEARCH_INDEX_SCHEMA_VERSION = "2"; const RRF_K = 60; const enum SearchTier { @@ -112,9 +115,7 @@ export class SqliteCatalogSearch { if ( !this.hasTable("package_search") || !this.hasTable("resource_search") || - !this.hasTable("document_chunk_search") || !this.hasTable("search_exact_names") || - !this.hasTable("search_document_reference_counts") || !this.hasTable("search_index_metadata") ) { return false; @@ -124,7 +125,12 @@ export class SqliteCatalogSearch { "SELECT value FROM search_index_metadata WHERE key = ? LIMIT 1", ) .get("schema_version"); - return row?.value === SEARCH_INDEX_SCHEMA_VERSION; + if (row?.value === METADATA_SEARCH_INDEX_SCHEMA_VERSION) return true; + return ( + row?.value === DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION && + this.hasTable("document_chunk_search") && + this.hasTable("search_document_reference_counts") + ); } searchPackages(input: SearchPackagesInput): SearchPackagesOutput { @@ -596,7 +602,21 @@ export class SqliteCatalogSearch { } private hasDocumentSearchRelations(): boolean { - return this.hasTable("spec_document_refs") && this.hasTable("spec_documents"); + return ( + this.searchIndexSchemaVersion() === DOCUMENT_SEARCH_INDEX_SCHEMA_VERSION && + this.hasTable("spec_document_refs") && + this.hasTable("spec_documents") && + this.hasTable("document_chunk_search") && + this.hasTable("search_document_reference_counts") + ); + } + + private searchIndexSchemaVersion(): string | undefined { + return this.db + .query<{ value: string }, []>( + "SELECT value FROM search_index_metadata WHERE key = 'schema_version' LIMIT 1", + ) + .get()?.value; } private hasTable(name: string): boolean { diff --git a/packages/core/src/portable.ts b/packages/core/src/portable.ts new file mode 100644 index 0000000..83c6758 --- /dev/null +++ b/packages/core/src/portable.ts @@ -0,0 +1,18 @@ +export * from "./ckan/client"; +export * from "./ckan/errors"; +export * from "./ckan/schemas"; +export * from "./ckan/types"; +export * from "./catalog/errors"; +export * from "./catalog/catalog-mappers"; +export * from "./catalog/platform-api"; +export * from "./catalog/resource-fetch"; +export * from "./catalog/search-index-schema"; +export * from "./catalog/schemas"; +export * from "./catalog/service"; +export * from "./catalog/sqlite-rows"; +export * from "./catalog/types"; +export * from "./sync/errors"; +export * from "./sync/records"; +export * from "./sync/schema"; +export * from "./sync/service"; +export * from "./sync/types"; diff --git a/packages/core/src/sync/records.ts b/packages/core/src/sync/records.ts new file mode 100644 index 0000000..d7d5e63 --- /dev/null +++ b/packages/core/src/sync/records.ts @@ -0,0 +1,160 @@ +import type { CkanGroup, CkanPackage, CkanResource } from "../ckan/types"; +import type { RetrievalKind } from "../catalog/types"; + +export interface CatalogPackageRecord { + readonly id: string; + readonly name: string; + readonly title: string | null; + readonly notes: string | null; + readonly url: string | null; + readonly updateFrequency: string | null; + readonly metadataCreated: string | null; + readonly metadataModified: string | null; + readonly organizationId: string | null; + readonly organizationName: string | null; + readonly organizationTitle: string | null; + readonly dataDictionary: string | null; + readonly sources: string | null; + readonly reference: string | null; + readonly supplementaryDocs: string | null; + readonly apiSpec: string | null; + readonly announcement: string | null; + readonly resourceCount: number; + readonly rawJson: string; +} + +export interface CatalogResourceRecord { + readonly packageName: string; + readonly id: string; + readonly sharedId: string | null; + readonly name: string | null; + readonly description: string | null; + readonly url: string | null; + readonly format: string | null; + readonly isApi: string | null; + readonly retrievalKind: RetrievalKind; + readonly retrievalReason: string; + readonly language: string | null; + readonly created: string | null; + readonly metadataModified: string | null; + readonly schema: string | null; + readonly supplementaryDocs: string | null; + readonly apiSpec: string | null; + readonly rawJson: string; +} + +export interface CatalogGroupRecord { + readonly id: string; + readonly name: string; + readonly title: string | null; + readonly displayName: string | null; + readonly description: string | null; + readonly packageCount: number | null; + readonly includedPackageCount: number | null; + readonly rawJson: string; +} + +export function normalizeCatalogPackage(packageValue: CkanPackage): CatalogPackageRecord { + const organization = packageValue.organization; + return { + id: packageValue.id, + name: packageValue.name, + title: packageValue.title, + notes: packageValue.notes, + url: packageValue.url, + updateFrequency: packageValue.update_frequency, + metadataCreated: packageValue.metadata_created, + metadataModified: packageValue.metadata_modified, + organizationId: organization?.id ?? null, + organizationName: organization?.name ?? null, + organizationTitle: organization?.title ?? null, + dataDictionary: catalogTextField(packageValue.data_dictionary), + sources: catalogTextField(packageValue.sources), + reference: catalogTextField(packageValue.references), + supplementaryDocs: catalogTextField(packageValue.supplementaryDocs), + apiSpec: catalogApiSpecField(packageValue.supplementaryDocs), + announcement: catalogTextField(packageValue.announcement), + resourceCount: packageValue.resources.length, + rawJson: JSON.stringify(packageValue), + }; +} + +export function normalizeCatalogResource( + packageName: string, + resource: CkanResource, +): CatalogResourceRecord { + const retrieval = classifyCatalogResource(resource); + return { + packageName, + id: resource.id, + sharedId: resource.shared_id, + name: resource.name, + description: resource.description, + url: resource.url, + format: resource.format, + isApi: resource.is_api, + retrievalKind: retrieval.kind, + retrievalReason: retrieval.reason, + language: resource.inLanguage ?? null, + created: resource.created, + metadataModified: resource.metadata_modified, + schema: catalogTextField(resource.schema), + supplementaryDocs: catalogTextField(resource.supplementaryDocs), + apiSpec: catalogApiSpecField(resource.supplementaryDocs), + rawJson: JSON.stringify(resource), + }; +} + +export function normalizeCatalogGroup(group: CkanGroup): CatalogGroupRecord { + return { + id: group.id, + name: group.name, + title: group.title, + displayName: group.display_name, + description: group.description, + packageCount: group.package_count, + includedPackageCount: group.packages?.length ?? null, + rawJson: JSON.stringify(group), + }; +} + +export function catalogTextField(value: unknown): string | null { + if (value === undefined || value === null) return null; + return typeof value === "string" ? value : JSON.stringify(value); +} + +export function catalogApiSpecField(value: unknown): string | null { + const text = catalogTextField(value); + if (!text) return null; + const normalized = text.toLowerCase(); + return normalized.includes("api") && + (normalized.includes("spec") || + normalized.includes("documentation") || + normalized.includes("document")) + ? text + : null; +} + +export function classifyCatalogResource(resource: CkanResource): { + readonly kind: RetrievalKind; + readonly reason: string; +} { + const format = resource.format?.toUpperCase() ?? ""; + const url = resource.url?.toLowerCase() ?? ""; + if (resource.is_api === "Y" || format === "API") { + return { kind: "api-resource", reason: "is_api or format marks this resource as API-like." }; + } + if (["CSV", "XLS", "XLSX"].includes(format)) { + return { kind: "tabular-file", reason: "The declared format is tabular." }; + } + if (["GEOJSON", "GML", "KML", "KMZ", "SHP", "GEOTIFF", "FGDB", "GTFS"].includes(format)) { + return { kind: "geospatial-file", reason: "The declared format is geospatial." }; + } + if (["ZIP", "7Z", "RAR"].includes(format) || url.endsWith(".zip")) { + return { kind: "archive-file", reason: "The resource appears to be an archive." }; + } + if (resource.url) { + return { kind: "direct-download", reason: "The resource has a direct URL." }; + } + return { kind: "unknown", reason: "The resource has insufficient retrieval metadata." }; +} diff --git a/packages/core/src/sync/sqlite.ts b/packages/core/src/sync/sqlite.ts index 4aab97e..09d64cf 100644 --- a/packages/core/src/sync/sqlite.ts +++ b/packages/core/src/sync/sqlite.ts @@ -2,8 +2,13 @@ import { Database } from "bun:sqlite"; import { Effect, Layer } from "effect"; import type { CkanGroup, CkanPackage, CkanResource } from "../ckan/types"; -import type { RetrievalKind } from "../catalog/types"; +import { METADATA_SEARCH_INDEX_SCHEMA_VERSION } from "../catalog/search-index-schema"; import { CatalogSyncStoreError } from "./errors"; +import { + normalizeCatalogGroup, + normalizeCatalogPackage, + normalizeCatalogResource, +} from "./records"; import { CATALOG_SCHEMA_STATEMENTS } from "./schema"; import { CatalogSyncStore } from "./service"; import type { CatalogSyncCounts, CatalogSyncRun, CatalogSyncStoreService } from "./types"; @@ -84,15 +89,16 @@ export class SqliteCatalogSyncStore implements CatalogSyncStoreService { last_seen_run = excluded.last_seen_run`, ); for (const group of groups) { + const record = normalizeCatalogGroup(group); statement.run( - group.id, - group.name, - group.title, - group.display_name, - group.description, - group.package_count, - group.packages?.length ?? null, - JSON.stringify(group), + record.id, + record.name, + record.title, + record.displayName, + record.description, + record.packageCount, + record.includedPackageCount, + record.rawJson, runId, ); } @@ -163,7 +169,7 @@ export class SqliteCatalogSyncStore implements CatalogSyncStoreService { } private writePackage(runId: string, packageValue: CkanPackage): void { - const organization = packageValue.organization; + const record = normalizeCatalogPackage(packageValue); this.db .query( `INSERT INTO packages ( @@ -194,25 +200,25 @@ export class SqliteCatalogSyncStore implements CatalogSyncStoreService { last_seen_run = excluded.last_seen_run`, ) .run( - packageValue.id, - packageValue.name, - packageValue.title, - packageValue.notes, - packageValue.url, - packageValue.update_frequency, - packageValue.metadata_created, - packageValue.metadata_modified, - organization?.id ?? null, - organization?.name ?? null, - organization?.title ?? null, - textField(packageValue.data_dictionary), - textField(packageValue.sources), - textField(packageValue.references), - textField(packageValue.supplementaryDocs), - apiSpecField(packageValue.supplementaryDocs), - textField(packageValue.announcement), - packageValue.resources.length, - JSON.stringify(packageValue), + record.id, + record.name, + record.title, + record.notes, + record.url, + record.updateFrequency, + record.metadataCreated, + record.metadataModified, + record.organizationId, + record.organizationName, + record.organizationTitle, + record.dataDictionary, + record.sources, + record.reference, + record.supplementaryDocs, + record.apiSpec, + record.announcement, + record.resourceCount, + record.rawJson, runId, ); @@ -231,7 +237,7 @@ export class SqliteCatalogSyncStore implements CatalogSyncStoreService { } private writeResource(runId: string, packageName: string, resource: CkanResource): void { - const retrieval = classifyResource(resource); + const record = normalizeCatalogResource(packageName, resource); this.db .query( `INSERT INTO resources ( @@ -241,23 +247,23 @@ export class SqliteCatalogSyncStore implements CatalogSyncStoreService { ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .run( - packageName, - resource.id, - resource.shared_id, - resource.name, - resource.description, - resource.url, - resource.format, - resource.is_api, - retrieval.kind, - retrieval.reason, - resource.inLanguage ?? null, - resource.created, - resource.metadata_modified, - textField(resource.schema), - textField(resource.supplementaryDocs), - apiSpecField(resource.supplementaryDocs), - JSON.stringify(resource), + record.packageName, + record.id, + record.sharedId, + record.name, + record.description, + record.url, + record.format, + record.isApi, + record.retrievalKind, + record.retrievalReason, + record.language, + record.created, + record.metadataModified, + record.schema, + record.supplementaryDocs, + record.apiSpec, + record.rawJson, runId, ); } @@ -313,7 +319,7 @@ export class SqliteCatalogSyncStore implements CatalogSyncStoreService { const resourceCount = this.db .query<{ count: number }, []>("SELECT COUNT(*) AS count FROM resources") .get()?.count; - metadata.run("schema_version", "3"); + metadata.run("schema_version", METADATA_SEARCH_INDEX_SCHEMA_VERSION); metadata.run("built_at", finishedAt); metadata.run("package_count", String(packageCount ?? counts.packages)); metadata.run("resource_count", String(resourceCount ?? counts.resources)); @@ -340,44 +346,3 @@ export const makeSqliteCatalogSyncStoreLayer = ( (store) => Effect.sync(() => store.close()), ), ); - -function textField(value: unknown): string | null { - if (value === undefined || value === null) return null; - return typeof value === "string" ? value : JSON.stringify(value); -} - -function apiSpecField(value: unknown): string | null { - const text = textField(value); - if (!text) return null; - const normalized = text.toLowerCase(); - return normalized.includes("api") && - (normalized.includes("spec") || - normalized.includes("documentation") || - normalized.includes("document")) - ? text - : null; -} - -function classifyResource(resource: CkanResource): { - readonly kind: RetrievalKind; - readonly reason: string; -} { - const format = resource.format?.toUpperCase() ?? ""; - const url = resource.url?.toLowerCase() ?? ""; - if (resource.is_api === "Y" || format === "API") { - return { kind: "api-resource", reason: "is_api or format marks this resource as API-like." }; - } - if (["CSV", "XLS", "XLSX"].includes(format)) { - return { kind: "tabular-file", reason: "The declared format is tabular." }; - } - if (["GEOJSON", "GML", "KML", "KMZ", "SHP", "GEOTIFF", "FGDB", "GTFS"].includes(format)) { - return { kind: "geospatial-file", reason: "The declared format is geospatial." }; - } - if (["ZIP", "7Z", "RAR"].includes(format) || url.endsWith(".zip")) { - return { kind: "archive-file", reason: "The resource appears to be an archive." }; - } - if (resource.url) { - return { kind: "direct-download", reason: "The resource has a direct URL." }; - } - return { kind: "unknown", reason: "The resource has insufficient retrieval metadata." }; -} diff --git a/packages/core/test/sync/sqlite.test.ts b/packages/core/test/sync/sqlite.test.ts index 5bb3f44..ff59c83 100644 --- a/packages/core/test/sync/sqlite.test.ts +++ b/packages/core/test/sync/sqlite.test.ts @@ -5,6 +5,8 @@ import { join } from "node:path"; import { Effect } from "effect"; import type { CkanGroup, CkanPackage } from "../../src/ckan/types"; +import { METADATA_SEARCH_INDEX_SCHEMA_VERSION } from "../../src/catalog/search-index-schema"; +import { SqliteCatalogService } from "../../src/catalog/sqlite"; import { SqliteCatalogSyncStore } from "../../src/sync/sqlite"; const temporaryDirectories: string[] = []; @@ -51,6 +53,24 @@ describe("SqliteCatalogSyncStore", () => { expect(store.db.query("SELECT COUNT(*) AS count FROM package_search").get()).toEqual({ count: 2, }); + expect( + store.db + .query<{ value: string }, []>( + "SELECT value FROM search_index_metadata WHERE key = 'schema_version'", + ) + .get(), + ).toEqual({ value: METADATA_SEARCH_INDEX_SCHEMA_VERSION }); + + const catalog = new SqliteCatalogService({ dbPath: join(directory, "catalog.sqlite") }); + try { + expect(catalog.hasSearchIndex()).toBe(true); + const search = await Effect.runPromise( + catalog.searchPackages({ query: "Traffic", limit: 10 }), + ); + expect(search.items.map((item) => item.package_id)).toContain("traffic"); + } finally { + catalog.close(); + } const partialRun = { ...run, id: "run-2", startedAt: "2026-08-08T00:02:00.000Z" }; await Effect.runPromise(store.begin(partialRun)); diff --git a/packages/hk-open-data/README.md b/packages/hk-open-data/README.md index 365ba02..63630d6 100644 --- a/packages/hk-open-data/README.md +++ b/packages/hk-open-data/README.md @@ -16,4 +16,8 @@ Library entrypoint: import { createCatalogService, syncCatalog } from "hk-open-data"; ``` +Use `hk-open-data/portable` for Effect services, schemas, API clients, synchronization contracts, and normalized storage records in runtimes that do not provide `bun:sqlite`. + +Catalog commands synchronize a missing or stale local catalog automatically. Set `HK_OPEN_DATA_AUTO_SYNC=0` to disable that behavior and use `sync` explicitly. + The package includes no hosted-service transport. Private services can implement the exported sync-store and catalog contracts on their own infrastructure. diff --git a/packages/hk-open-data/package.json b/packages/hk-open-data/package.json index 8389d98..aa8871b 100644 --- a/packages/hk-open-data/package.json +++ b/packages/hk-open-data/package.json @@ -38,6 +38,10 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" + }, + "./portable": { + "types": "./dist/portable.d.ts", + "import": "./dist/portable.js" } }, "publishConfig": { diff --git a/packages/hk-open-data/scripts/build.ts b/packages/hk-open-data/scripts/build.ts index 64faa3c..2059b66 100644 --- a/packages/hk-open-data/scripts/build.ts +++ b/packages/hk-open-data/scripts/build.ts @@ -13,6 +13,7 @@ rmSync(outdir, { recursive: true, force: true }); mkdirSync(outdir, { recursive: true }); await build(resolve(packageRoot, "../core/src/index.ts"), "index.js", true); +await build(resolve(packageRoot, "../core/src/portable.ts"), "portable.js", true); await build(resolve(packageRoot, "../cli/src/index.ts"), "cli.js"); const declarations = Bun.spawnSync( diff --git a/packages/hk-open-data/src/portable.ts b/packages/hk-open-data/src/portable.ts new file mode 100644 index 0000000..1c1ce78 --- /dev/null +++ b/packages/hk-open-data/src/portable.ts @@ -0,0 +1 @@ +export * from "../../core/src/portable"; diff --git a/scripts/README.md b/scripts/README.md index 6676713..a5192a1 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,7 +2,7 @@ ## Catalog synchronization -CKAN metadata ingestion is implemented in TypeScript and shared with cloud adapters: +CKAN metadata ingestion is implemented in TypeScript and shared with cloud adapters. Ordinary CLI catalog commands run this synchronization automatically when the database is missing or older than 24 hours; set `HK_OPEN_DATA_AUTO_SYNC=0` to opt out: ```sh bun run sync -- --db data/catalog.sqlite @@ -117,7 +117,7 @@ Builds parsed-document SQLite FTS5 indexes after document parsing. The TypeScrip uv run python scripts/build_search_index.py --db data/catalog.sqlite ``` -The build is atomic. It indexes each parsed document once and preserves package/resource scope through `spec_document_refs`. +The build is atomic. It indexes each parsed document once and preserves package/resource scope through `spec_document_refs`. TypeScript synchronization maintains a metadata-only FTS index; this Python build upgrades it to the document-aware index when parsed documents are available. ## `analyze_spec_text_tables.py` diff --git a/scripts/prepare-effect.sh b/scripts/prepare-effect.sh deleted file mode 100755 index 24a9d31..0000000 --- a/scripts/prepare-effect.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env sh - -set -eu - -repo_dir=".repos/effect" -repo_url="https://github.com/Effect-TS/effect-smol" - -if [ -d "$repo_dir/.git" ]; then - exit 0 -fi - -mkdir -p ".repos" -git clone "$repo_url" "$repo_dir"