From dd3d8ca9dddf506588f532cd14a0b1cdd0f5957c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9rick=20Madore?=
<93156843+fmadore@users.noreply.github.com>
Date: Sat, 1 Aug 2026 10:28:24 +0200
Subject: [PATCH] Implement visualization improvement issues
---
.github/workflows/ci.yml | 26 +-
.github/workflows/embeddings.yml | 117 +++
.github/workflows/release.yml | 3 +-
.github/workflows/wordclouds.yml | 4 +-
.gitignore | 6 +
CHANGELOG.md | 28 +
CONTRIBUTING.md | 15 +-
Module.php | 22 +
README.md | 17 +-
asset/css/dre-visualizations.css | 267 +++++
asset/js/dashboard-charts-wordcloud.js | 22 +-
asset/js/dashboard-charts.bundle.js | 31 +-
asset/js/dashboard-layouts.js | 9 +-
asset/js/semantic-map.js | 260 +++++
asset/js/semantic-similar.js | 80 ++
config/amira-profile.json | 38 +
config/module.config.php | 2 +
config/module.ini | 4 +-
docs/ADMINISTRATION.md | 11 +-
docs/ARCHITECTURE.md | 13 +-
docs/SECURITY_AND_PRIVACY.md | 6 +
docs/SEMANTIC_EMBEDDINGS.md | 75 ++
package.json | 6 +-
scripts/check-release-metadata.mjs | 24 +-
src/Controller/Site/EmbedController.php | 5 +
.../Aggregators/MediaChartsTrait.php | 45 +-
src/Precompute/AmiraProfile.php | 52 +-
src/Precompute/Runner.php | 24 +
src/Site/BlockLayout/SemanticMap.php | 28 +
.../ResourcePageBlockLayout/SimilarItems.php | 26 +
src/View/Helper/DashboardAssets.php | 16 +
tests/AggregatorsTest.php | 24 +
tests/AmiraProfileTest.php | 15 +
tests/browser/semantic-fixture.html | 78 ++
tools/embeddings/build_embeddings.py | 937 ++++++++++++++++++
tools/embeddings/requirements.txt | 5 +
.../embeddings/tests/test_build_embeddings.py | 257 +++++
view/common/block-layout/semantic-map.phtml | 28 +
.../similar-items.phtml | 28 +
39 files changed, 2606 insertions(+), 48 deletions(-)
create mode 100644 .github/workflows/embeddings.yml
create mode 100644 asset/js/semantic-map.js
create mode 100644 asset/js/semantic-similar.js
create mode 100644 docs/SEMANTIC_EMBEDDINGS.md
create mode 100644 src/Site/BlockLayout/SemanticMap.php
create mode 100644 src/Site/ResourcePageBlockLayout/SimilarItems.php
create mode 100644 tests/browser/semantic-fixture.html
create mode 100644 tools/embeddings/build_embeddings.py
create mode 100644 tools/embeddings/requirements.txt
create mode 100644 tools/embeddings/tests/test_build_embeddings.py
create mode 100644 view/common/block-layout/semantic-map.phtml
create mode 100644 view/common/resource-page-block-layout/similar-items.phtml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3bb939df..68a754c4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,6 +1,7 @@
name: CI
on:
+ workflow_dispatch:
pull_request:
push:
branches: [main]
@@ -16,8 +17,8 @@ jobs:
browser-contracts:
runs-on: ubuntu-latest
steps:
- # actions/checkout v7.0.0, verified upstream 2026-06-18.
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+ # actions/checkout v7.0.1, verified upstream 2026-07-31.
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
- run: npm run check
php:
@@ -27,7 +28,7 @@ jobs:
matrix:
php: ['8.2', '8.3', '8.4', '8.5']
steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
# shivammathur/setup-php 2.37.2, verified upstream 2026-06-08.
- uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240
with:
@@ -52,7 +53,7 @@ jobs:
module-contract:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
- uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240
with:
php-version: '8.4'
@@ -66,3 +67,20 @@ jobs:
unzip -q "$RUNNER_TEMP/omeka-s.zip" -d "$RUNNER_TEMP"
- name: Module classes declare against Omeka S
run: php scripts/check-module-contract.php "$RUNNER_TEMP/omeka-s"
+
+ embeddings:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ # actions/setup-python v6.3.0, verified upstream 2026-07-31.
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+ with:
+ python-version: '3.12'
+ cache: pip
+ cache-dependency-path: tools/embeddings/requirements.txt
+ - name: Install pinned embedding dependencies
+ run: python -m pip install -r tools/embeddings/requirements.txt
+ - name: Validate profile and embedding contracts
+ run: |
+ python tools/embeddings/build_embeddings.py --validate-profile
+ python -m unittest discover -s tools/embeddings/tests -v
diff --git a/.github/workflows/embeddings.yml b/.github/workflows/embeddings.yml
new file mode 100644
index 00000000..72e8857a
--- /dev/null
+++ b/.github/workflows/embeddings.yml
@@ -0,0 +1,117 @@
+name: Build semantic embeddings
+
+# Manual because the Gemini call is a paid, secret-backed precompute. Ordinary
+# pull-request CI tests every pure contract without a key; this workflow fetches
+# only the unauthenticated public Omeka API, incrementally refreshes vectors,
+# opens a bot PR for the compact artifacts, and publishes full vectors as a
+# versioned GitHub Release rather than committing them.
+on:
+ workflow_dispatch:
+ inputs:
+ scope:
+ description: Embedding scope
+ required: true
+ type: choice
+ default: missing
+ options:
+ - missing
+ - all
+ publish_vectors:
+ description: Publish the full-vector GitHub Release
+ required: true
+ type: boolean
+ default: true
+
+permissions:
+ contents: write
+ pull-requests: write
+
+concurrency:
+ group: semantic-embeddings
+ cancel-in-progress: false
+
+env:
+ GEMINI_EMBEDDING_MODEL: gemini-embedding-2
+ GEMINI_EMBEDDING_DIMS: '768'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ # actions/checkout v7.0.1, verified upstream 2026-07-31.
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+ # actions/setup-python v6.3.0, verified upstream 2026-07-31.
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+ with:
+ python-version: '3.12'
+ cache: pip
+ cache-dependency-path: tools/embeddings/requirements.txt
+
+ # actions/cache v6.1.0, verified upstream 2026-07-31. A unique write key
+ # restores the newest compatible prefix and persists the refreshed cache.
+ - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
+ with:
+ path: asset/data/embeddings/cache.json
+ key: embeddings-${{ runner.os }}-${{ env.GEMINI_EMBEDDING_MODEL }}-${{ env.GEMINI_EMBEDDING_DIMS }}-${{ github.run_id }}
+ restore-keys: |
+ embeddings-${{ runner.os }}-${{ env.GEMINI_EMBEDDING_MODEL }}-${{ env.GEMINI_EMBEDDING_DIMS }}-
+
+ - name: Install pinned dependencies
+ run: python -m pip install -r tools/embeddings/requirements.txt
+
+ - name: Run embedding contract tests
+ run: |
+ python tools/embeddings/build_embeddings.py --validate-profile
+ python -m unittest discover -s tools/embeddings/tests -v
+
+ - name: Build public semantic artifacts
+ env:
+ GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+ run: python tools/embeddings/build_embeddings.py --scope "${{ inputs.scope }}"
+
+ - name: Validate generated artifact set
+ run: python tools/embeddings/build_embeddings.py --validate-artifacts
+
+ # Save even when a later API batch fails: build_embeddings.py flushes each
+ # paid successful batch, so the next run can resume instead of rebilling it.
+ - name: Persist the incremental embedding cache
+ if: always()
+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
+ with:
+ path: asset/data/embeddings/cache.json
+ key: embeddings-${{ runner.os }}-${{ env.GEMINI_EMBEDDING_MODEL }}-${{ env.GEMINI_EMBEDDING_DIMS }}-${{ github.run_id }}
+
+ - name: Open bot PR for derived map and recommendations
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git add asset/data/embeddings/map.json asset/data/embeddings/similar.json asset/data/embeddings/report.json
+ if git diff --staged --quiet; then
+ echo "No semantic artifact changes."
+ else
+ branch="automation/embeddings-${GITHUB_RUN_ID}"
+ git switch -c "$branch"
+ git commit -m "Semantic embeddings: refresh public derived artifacts"
+ git push --set-upstream origin "$branch"
+ gh pr create --base main --head "$branch" \
+ --title "Semantic embeddings: refresh public artifacts" \
+ --body "Automated public-only semantic-map and recommendation refresh from workflow run ${GITHUB_RUN_ID}."
+ fi
+
+ - name: Publish versioned full-vector release
+ if: ${{ inputs.publish_vectors }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ tag="semantic-embeddings-v1-${GITHUB_RUN_NUMBER}"
+ gh release create "$tag" \
+ asset/data/embeddings/release/vectors.f32 \
+ asset/data/embeddings/release/ids.json \
+ asset/data/embeddings/release/manifest.json \
+ --target "$GITHUB_SHA" \
+ --title "AMIRA semantic embeddings v1 · build ${GITHUB_RUN_NUMBER}" \
+ --notes "Public AMIRA vectors. Read manifest.json before indexing; embedding spaces are model-specific." \
+ --latest=false
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index aefc8782..aa8427b1 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -15,7 +15,8 @@ jobs:
release:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+ # actions/checkout v7.0.1, verified upstream 2026-07-31.
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
fetch-depth: 0
- uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240
diff --git a/.github/workflows/wordclouds.yml b/.github/workflows/wordclouds.yml
index c2c21882..3d365cd7 100644
--- a/.github/workflows/wordclouds.yml
+++ b/.github/workflows/wordclouds.yml
@@ -19,8 +19,8 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- # actions/checkout v7.0.0, verified upstream 2026-06-18.
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+ # actions/checkout v7.0.1, verified upstream 2026-07-31.
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
# actions/setup-python v6.3.0, verified upstream 2026-06-24.
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
diff --git a/.gitignore b/.gitignore
index eaa2c006..f2389605 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ node_modules/
.idea/
.vscode/
__pycache__/
+.codex/
# Per-item knowledge-graph JSON (~6,000 files, ~188 MB) is regenerated in-Omeka
# via the admin "Regenerate now" job — not committed. The front-end falls back to
@@ -33,3 +34,8 @@ asset/data/communities/
# Additional install-specific outputs from the same precompute job.
asset/data/network-explorer.json
asset/data/featured-collections/
+
+# Semantic embeddings: compact public map/recommendation/report JSON is
+# committed, while the full-vector cache and release payload stay out of Git.
+asset/data/embeddings/cache.json
+asset/data/embeddings/release/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b912a58..c3f432ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,34 @@
All notable changes are documented here. Versions follow Semantic Versioning.
+## 2.24.0 — 2026-07-31
+
+### Added
+
+- **One multilingual semantic space across six public corpora.** The manual,
+ secret-backed embedding workflow creates uniformly bounded cards for podcasts,
+ YouTube videos, publications, projects, research sections, and research items;
+ incrementally embeds changed cards with `gemini-embedding-2`; projects them with
+ deterministic UMAP; reports quality/low-signal coverage; commits the compact
+ map and recommendation contracts; and publishes normalized 768-dimensional
+ float32 vectors as a versioned GitHub Release.
+- **Semantic Map** site-page block and **Similar Items** resource-page block.
+ The map supports title search, resource-type/cluster colouring, accessible
+ controls, lazy ECharts loading, and embedding. Recommendations are progressive
+ enhancement and never surface low-signal records.
+- CI now validates the six-corpus profile, card construction, public filtering,
+ incremental cache behaviour, recommendation eligibility, and vector release
+ schema alongside the PHP and browser-contract matrices.
+
+### Changed
+
+- Podcasts now include linked subjects, subject trends/co-occurrence, locations,
+ and an items-by-country choropleth. Transcript word clouds expose translated,
+ accessible language controls, while the PHP fallback performs a curated layer
+ of English/French inflection folding when the spaCy-built input is unavailable.
+- Project and installation metadata use the canonical **DRE-Visualizations**
+ repository name throughout.
+
## 2.23.0 — 2026-07-30
### Fixed
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9f645958..cc34fe13 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -24,6 +24,7 @@ the point; a static network is cheaper as an ECharts series.
npm run check # design-token contract + JS syntax sweep + registry/layout/embed contracts
docker run --rm -v "$PWD:/m" php:8.4-cli php /m/tests/AggregatorsTest.php # aggregator regressions
docker run --rm -v "$PWD:/m" php:8.4-cli php /m/tests/KnowledgeGraphsTest.php # graph builder regressions
+python -m unittest discover -s tools/embeddings/tests -v # embedding contracts
```
The aggregators are dependency-free and unit-tested — add a mock-data case for
@@ -139,24 +140,28 @@ All visualizations load from precomputed JSON under `asset/data/`:
asset/data/
├── geo/countries.geojson # Natural Earth boundaries (choropleth) — committed INPUT
├── wordclouds/ # Lemmatised frequencies from the CI Action — committed INPUT
+├── embeddings/ # Committed map/recommendations/report; vectors are release-only
├── communities/ # Multi-entity co-occurrence network (baked FA2 positions)
├── knowledge-graphs/ # One per item — gitignored, regenerated in-Omeka
├── photo-galleries/ # One per image-bearing item set — gitignored
└── item-dashboards/ # Dashboards + {type}-index.json + publications/podcasts/youtube/…
```
-**Everything** regenerates inside Omeka via the admin **"Regenerate now"** button —
+Installation-specific dashboard data regenerates inside Omeka via the admin **"Regenerate now"** button —
a pure-PHP engine under `src/Precompute/` (`DataLoader` → `Aggregators` /
-`KnowledgeGraphs` → `Runner`) that reuses Omeka's own database connection. No
-Python, shell access, or extra credentials — the module ships **zero** Python. The
+`KnowledgeGraphs` → `Runner`) that reuses Omeka's own database connection. It needs no
+Python, shell access, or extra credentials at runtime. The
knowledge-graph JSON (~6,000 files) is **not committed**; until the first run the
front-end falls back to a lighter live REST-API graph.
-Two static **inputs** are the exception, produced outside Omeka and committed like
+Static **inputs** are the exception, produced outside Omeka and committed like
`countries.geojson`: the `wordclouds/` frequencies come from the **Build word
clouds** GitHub Action (`tools/wordclouds/build_wordclouds.py`, spaCy
lemmatisation — PHP can't do it), read back by `Runner::wordCloudInput()` with an
-in-PHP tokeniser fallback.
+in-PHP tokeniser fallback. The **Build semantic embeddings** Action runs the
+secret-backed Gemini precompute: only `map.json`, `similar.json`, and `report.json`
+are committed, while full vectors and their manifest are published as a versioned
+release. Its consumer contract is documented in `docs/SEMANTIC_EMBEDDINGS.md`.
The JS is modular — one vanilla-JS IIFE per concern (chart builders, controllers,
the `window.RV` core, registry, layouts). The authoritative per-file tree lives in
diff --git a/Module.php b/Module.php
index 91a768ea..2691df7a 100644
--- a/Module.php
+++ b/Module.php
@@ -269,6 +269,28 @@ public static function clientTranslations($view): array
'fullscreen' => $view->translate('Fullscreen'),
'exitFullscreen' => $view->translate('Exit fullscreen'),
'item' => $view->translate('item'),
+ 'language' => $view->translate('Language'),
+ 'words' => $view->translate('Words'),
+ 'langAll' => $view->translate('All'),
+ 'langEnglish' => $view->translate('English'),
+ 'langFrench' => $view->translate('French'),
+ 'langGerman' => $view->translate('German'),
+ 'langPortuguese' => $view->translate('Portuguese'),
+ 'semanticColorBy' => $view->translate('Colour by'),
+ 'semanticType' => $view->translate('Resource type'),
+ 'semanticCluster' => $view->translate('Semantic cluster'),
+ 'semanticSearch' => $view->translate('Find a record on the map'),
+ 'semanticSearchPlaceholder' => $view->translate('Search titles'),
+ 'semanticMapAria' => $view->translate('Semantic map of public collection records. Nearby points have similar metadata and descriptions.'),
+ 'semanticLowSignal' => $view->translate('Faint points have too little descriptive metadata for recommendations.'),
+ 'semanticLoadError' => $view->translate('The semantic map is not available yet. Run the embeddings workflow and try again.'),
+ 'semanticNoSearchResults' => $view->translate('No matching records.'),
+ 'semanticSimilarity' => $view->translate('similar'),
+ 'semanticMapTitle' => $view->translate('Semantic map'),
+ 'semanticMapIntro' => $view->translate('Nearby records use similar language, subjects, places, and descriptions. The map joins every resource type in one multilingual space.'),
+ 'semanticSharedSpace' => $view->translate('Shared semantic space'),
+ 'semanticRecords' => $view->translate('records'),
+ 'semanticLowSignalCount' => $view->translate('low-signal'),
];
}
diff --git a/README.md b/README.md
index 649218c5..bdc86984 100644
--- a/README.md
+++ b/README.md
@@ -183,8 +183,9 @@ YouTube videos carry no `dcterms:type` of their own, so they don't appear in the
Analytics for the cluster's curated **podcast episodes** — the manually-catalogued **Podcasts** item set configured as `itemSets.podcasts` in `config/amira-profile.json` (`fabio:AudioDocument`). Added as a **site-page block** (Admin > Sites > [site] > Pages), it loads `asset/data/item-dashboards/podcasts.json` and shows:
- **summary stat cards** — episodes, series, distinct **speakers** (`marcrel:spk`), total **hours of audio** (with the average length), and the languages — the same reusable component as the Collection Overview;
-- **transcript word cloud** — the headline chart, from the episodes' AI-generated transcripts (`bibo:content`), with audio cues (`[music]`), `Speaker N:` labels and numbers stripped. **Lemmatised** when the [Word clouds](#word-clouds-lemmatised) Action has run; the in-PHP tokeniser (`Aggregators::buildTranscriptWordCloud`, a tunable EN+FR stop-word/filler list) is the fallback;
-- **speakers & hosts** (`marcrel:spk` / `hst` / `sde`), the **episode-length** distribution (`dcterms:extent`, ISO-8601, bucketed into bands by `Aggregators::buildDurationHistogram`), **episodes by year** (`dcterms:date`), and **episodes by series** (`dcterms:isPartOf`, clickable through to each series).
+- **transcript word cloud** — the headline chart, from the episodes' AI-generated transcripts (`bibo:content`), with audio cues (`[music]`), `Speaker N:` labels and numbers stripped. **Lemmatised** when the [Word clouds](#word-clouds-lemmatised) Action has run, with an accessible **All / English / French / German / Portuguese** language switch. The in-PHP fallback also collapses common English and French inflections in addition to its EN+FR stop-word/filler filtering;
+- **speakers & hosts** (`marcrel:spk` / `hst` / `sde`), the **episode-length** distribution (`dcterms:extent`, ISO-8601, bucketed into bands by `Aggregators::buildDurationHistogram`), **episodes by year** (`dcterms:date`), and **episodes by series** (`dcterms:isPartOf`, clickable through to each series);
+- **subjects and places** — ranked facets, subject trends, a subject co-occurrence chord, and an items-by-country choropleth using the same linked-value and geocoding rules as the other collection dashboards.
Podcasts carry no `dcterms:type` of their own, so (like YouTube videos) they don't appear in the resource-type pie *here*; instead they fold into the **Collection Overview** under a single synthetic **Podcast** type (see above). Speakers and series are clickable through to their Omeka pages.
@@ -197,6 +198,12 @@ The text word clouds (Podcasts transcripts, Publications abstracts, YouTube capt
- These are committed **static inputs** — like `geo/countries.geojson`, *not* the git-ignored generated dashboards. The precompute reads them via `Runner::wordCloudInput()` and folds the combined (`all`) frequencies into the dashboard; when a file is absent it **falls back** to the in-PHP tokeniser, so the clouds always render — just unlemmatised until the Action has run.
- **Reusable:** add a corpus under `wordcloudCorpora` in `config/amira-profile.json` (item-set key + text property). The Python builder and PHP precompute share that profile, and the per-language buckets feed the word cloud's **language toggle** (shipped in v2.16.0).
+### Semantic Map & Similar Items
+
+The **Semantic Map** site-page block (Admin > Sites > [site] > Pages) places public podcasts, YouTube videos, publications, projects, research sections, and research items in one multilingual Gemini embedding space. Its UMAP scatter can be coloured by resource type or semantic cluster, searched by title, zoomed, and embedded like the other site blocks. Low-signal records remain visible as faint context but do not produce recommendations.
+
+The **Similar Items** resource-page block (Admin > Sites > [site] > Theme > Configure resource pages) adds up to six cross-type neighbours to an item page and stays hidden when no reliable recommendation exists. Both components read compact, committed public-only JSON from `asset/data/embeddings/`; the full 768-dimensional float32 vectors are kept out of Git and published as a versioned GitHub Release for downstream search systems. See [Semantic embeddings](docs/SEMANTIC_EMBEDDINGS.md) for the shared card, schema, refresh, and compatibility contract.
+
### What's New
A recent-additions feed with a **3 / 6 / 12-month** window selector and a "most active projects" bar. Added as a **site-page block** (Admin > Sites > [site] > Pages), it loads `asset/data/item-dashboards/whats-new.json`. "Now" is the latest item-creation date in the corpus, so it stays meaningful regardless of when the data was imported.
@@ -261,12 +268,12 @@ Every embed shows a small **source** link back to the site, and the endpoint sen
Download via Omeka S CLI:
```bash
-docker compose exec php omeka-s-cli module:download --base-path /var/www/html https://github.com/AM-Digital-Research-Environment/ResourceVisualizations/releases/latest/download/DreVisualizations.zip
+docker compose exec php omeka-s-cli module:download --base-path /var/www/html https://github.com/AM-Digital-Research-Environment/DRE-Visualizations/releases/latest/download/DreVisualizations.zip
```
Then activate in **Admin > Modules**.
-> **Module folder name.** Omeka loads this module from a directory named `DreVisualizations`, matching the PHP namespace. Official release archives already contain that top-level directory. For development installs, clone the `ResourceVisualizations` repository explicitly into it: `git clone https://github.com/AM-Digital-Research-Environment/ResourceVisualizations.git modules/DreVisualizations`.
+> **Module folder name.** Omeka loads this module from a directory named `DreVisualizations`, matching the PHP namespace. Official release archives already contain that top-level directory. For development installs, clone the `DRE-Visualizations` repository explicitly into it: `git clone https://github.com/AM-Digital-Research-Environment/DRE-Visualizations.git modules/DreVisualizations`.
### Configure Resource Pages
@@ -302,7 +309,7 @@ Watch progress and any errors at **Admin → Jobs → the job's log**. Re-run af
To pull a new module **release** into the container:
```bash
-docker compose exec php omeka-s-cli module:download --base-path /var/www/html --force https://github.com/AM-Digital-Research-Environment/ResourceVisualizations/releases/latest/download/DreVisualizations.zip
+docker compose exec php omeka-s-cli module:download --base-path /var/www/html --force https://github.com/AM-Digital-Research-Environment/DRE-Visualizations/releases/latest/download/DreVisualizations.zip
docker compose restart php
```
diff --git a/asset/css/dre-visualizations.css b/asset/css/dre-visualizations.css
index e0098827..9300e745 100644
--- a/asset/css/dre-visualizations.css
+++ b/asset/css/dre-visualizations.css
@@ -98,6 +98,273 @@ body {
--rv-lift-sm: var(--lift-sm, -0.25rem); /* -4px */
}
+/* ------------------------------------------------------------------ */
+/* Semantic map + cross-type recommendations */
+/* ------------------------------------------------------------------ */
+
+.semantic-map-intro,
+.semantic-similar-intro {
+ max-width: 72ch;
+ margin: 0 0 var(--rv-space-5);
+ color: var(--rv-text-color);
+}
+
+.semantic-map-toolbar {
+ position: relative;
+ z-index: 2;
+ display: grid;
+ grid-template-columns: minmax(0, auto) minmax(16rem, 28rem);
+ align-items: end;
+ justify-content: space-between;
+ gap: var(--rv-space-4);
+ margin-bottom: var(--rv-space-4);
+}
+
+.semantic-map-mode {
+ display: inline-flex;
+ width: fit-content;
+ padding: var(--rv-space-1);
+ border: 1px solid var(--rv-border);
+ border-radius: var(--rv-radius-full);
+ background: var(--rv-bg-sunken);
+}
+
+button.semantic-map-mode__button {
+ min-height: 2.5rem;
+ padding: var(--rv-space-2) var(--rv-space-4);
+ border: 0;
+ border-radius: var(--rv-radius-full);
+ background: transparent;
+ box-shadow: none;
+ color: var(--rv-text-color);
+ font: inherit;
+ font-size: var(--rv-text-sm);
+ font-weight: 700;
+ cursor: pointer;
+ transition: background var(--rv-transition-fast), color var(--rv-transition-fast), box-shadow var(--rv-transition-fast);
+}
+
+button.semantic-map-mode__button:hover {
+ color: var(--rv-text-strong);
+}
+
+button.semantic-map-mode__button.is-active {
+ background: var(--rv-bg-raised);
+ box-shadow: var(--rv-shadow-sm);
+ color: var(--rv-accent);
+}
+
+button.semantic-map-mode__button:focus-visible {
+ outline: none;
+ box-shadow: var(--rv-focus-ring);
+}
+
+.semantic-map-search {
+ position: relative;
+}
+
+.semantic-map-search__label {
+ display: grid;
+ gap: var(--rv-space-1);
+ color: var(--rv-text-strong);
+ font-size: var(--rv-text-xs);
+ font-weight: 700;
+}
+
+.semantic-map-search__input {
+ width: 100%;
+ min-height: 2.75rem;
+ padding: var(--rv-space-2) var(--rv-space-3);
+ border: 1px solid var(--rv-border-strong);
+ border-radius: var(--rv-radius-sm);
+ background: var(--rv-bg);
+ color: var(--rv-text-strong);
+ font: inherit;
+ font-size: var(--rv-text-sm);
+}
+
+.semantic-map-search__input:focus-visible {
+ outline: none;
+ border-color: var(--rv-accent);
+ box-shadow: var(--rv-focus-ring);
+}
+
+.semantic-map-search__results {
+ position: absolute;
+ top: calc(100% + var(--rv-space-1));
+ right: 0;
+ left: 0;
+ z-index: 20;
+ max-height: 20rem;
+ overflow-y: auto;
+ margin: 0;
+ padding: var(--rv-space-1);
+ list-style: none;
+ border: 1px solid var(--rv-border);
+ border-radius: var(--rv-radius-sm);
+ background: var(--rv-bg-raised);
+ box-shadow: var(--rv-shadow);
+}
+
+.semantic-map-search__result {
+ display: grid;
+ grid-template-columns: minmax(0, 1fr) auto;
+ align-items: baseline;
+ gap: var(--rv-space-3);
+ border-radius: var(--rv-radius-sm);
+}
+
+.semantic-map-search__result a {
+ padding: var(--rv-space-2) var(--rv-space-3);
+ color: var(--rv-text-strong);
+ font-size: var(--rv-text-sm);
+ font-weight: 650;
+ text-decoration: none;
+}
+
+.semantic-map-search__result:has(a:hover),
+.semantic-map-search__result:has(a:focus-visible) {
+ background: var(--rv-bg-sunken);
+}
+
+.semantic-map-search__result a:focus-visible {
+ outline: none;
+ box-shadow: var(--rv-focus-ring);
+}
+
+.semantic-map-search__type {
+ padding-right: var(--rv-space-3);
+ color: var(--rv-text-color);
+ font-size: var(--rv-text-2xs);
+ white-space: nowrap;
+}
+
+.semantic-map-search__empty {
+ padding: var(--rv-space-3);
+ color: var(--rv-text-color);
+ font-size: var(--rv-text-sm);
+}
+
+.semantic-map-panel {
+ min-width: 0;
+}
+
+.chart-container.semantic-map-chart {
+ height: 38rem;
+ min-height: 28rem;
+}
+
+.semantic-map-status,
+.semantic-map-note {
+ margin: var(--rv-space-2) 0 0;
+ color: var(--rv-text-color);
+ font-size: var(--rv-text-xs);
+}
+
+.semantic-map-status {
+ color: var(--rv-text-strong);
+ font-weight: 700;
+}
+
+.semantic-similar-list {
+ margin: 0;
+ padding: 0;
+ list-style: none;
+ border-top: 1px solid var(--rv-border);
+}
+
+.semantic-similar-item {
+ border-bottom: 1px solid var(--rv-border-light);
+}
+
+.semantic-similar-link {
+ display: grid;
+ grid-template-columns: minmax(0, 1fr) auto;
+ align-items: center;
+ gap: var(--rv-space-4);
+ padding: var(--rv-space-3) var(--rv-space-2);
+ border-radius: var(--rv-radius-sm);
+ color: var(--rv-text-strong);
+ text-decoration: none;
+ transition: background var(--rv-transition-fast), color var(--rv-transition-fast);
+}
+
+.semantic-similar-link:hover {
+ background: var(--rv-bg-sunken);
+ color: var(--rv-accent);
+}
+
+.semantic-similar-link:focus-visible {
+ outline: none;
+ box-shadow: var(--rv-focus-ring);
+}
+
+.semantic-similar-copy {
+ display: grid;
+ justify-items: start;
+ gap: var(--rv-space-1);
+ min-width: 0;
+}
+
+.semantic-similar-type {
+ display: inline-block;
+ padding: 0.15em 0.65em;
+ border-radius: var(--rv-radius-full);
+ background: var(--rv-bg-sunken);
+ color: var(--rv-text-color);
+ font-size: var(--rv-text-2xs);
+ font-weight: 750;
+ line-height: 1.45;
+}
+
+.semantic-similar-title {
+ overflow-wrap: anywhere;
+ font-size: var(--rv-text-base);
+ font-weight: 700;
+ line-height: 1.35;
+}
+
+.semantic-similar-score {
+ min-width: 3.4rem;
+ color: var(--rv-accent);
+ font-variant-numeric: tabular-nums;
+ font-weight: 800;
+ text-align: right;
+}
+
+@media (max-width: 700px) {
+ .semantic-map-toolbar {
+ grid-template-columns: minmax(0, 1fr);
+ align-items: stretch;
+ }
+
+ .semantic-map-mode {
+ width: 100%;
+ }
+
+ button.semantic-map-mode__button {
+ flex: 1;
+ padding-inline: var(--rv-space-2);
+ }
+
+ .chart-container.semantic-map-chart {
+ height: 29rem;
+ min-height: 24rem;
+ }
+
+ .semantic-similar-link {
+ gap: var(--rv-space-2);
+ padding-inline: 0;
+ }
+}
+
+@media (prefers-reduced-motion: reduce) {
+ button.semantic-map-mode__button,
+ .semantic-similar-link {
+ transition: none;
+ }
+}
+
/*
* Dark fallbacks for NON-DRE host themes only. When the DRE theme is active it
* already supplies dark values for --surface / --border / --ink-* / --primary,
diff --git a/asset/js/dashboard-charts-wordcloud.js b/asset/js/dashboard-charts-wordcloud.js
index b38af299..98d6ceca 100644
--- a/asset/js/dashboard-charts-wordcloud.js
+++ b/asset/js/dashboard-charts-wordcloud.js
@@ -21,7 +21,13 @@
ns.charts = ns.charts || {};
- var LANG_NAMES = { en: 'English', fr: 'French', de: 'German', pt: 'Portuguese' };
+ var LANG_NAMES = {
+ all: ['langAll', 'All'],
+ en: ['langEnglish', 'English'],
+ fr: ['langFrench', 'French'],
+ de: ['langGerman', 'German'],
+ pt: ['langPortuguese', 'Portuguese']
+ };
var _wordCloudOk = null;
function isWordCloudAvailable() {
@@ -117,17 +123,25 @@
if (multi && langs.length > 1) {
var langBar = document.createElement('div');
langBar.className = 'rv-word-langs';
+ langBar.setAttribute('role', 'group');
+ langBar.setAttribute('aria-label', ns.t('language', 'Language'));
langs.forEach(function (code) {
var b = document.createElement('button');
b.type = 'button';
b.className = 'rv-word-lang' + (code === curLang ? ' is-active' : '');
- b.textContent = LANG_NAMES[code] || code.toUpperCase();
+ b.setAttribute('aria-pressed', code === curLang ? 'true' : 'false');
+ var langName = LANG_NAMES[code];
+ b.textContent = langName ? ns.t(langName[0], langName[1]) : code.toUpperCase();
b.addEventListener('click', function () {
if (code === curLang) return;
curLang = code;
entries = toEntries(rawFor());
- langBar.querySelectorAll('.rv-word-lang').forEach(function (x) { x.classList.remove('is-active'); });
+ langBar.querySelectorAll('.rv-word-lang').forEach(function (x) {
+ x.classList.remove('is-active');
+ x.setAttribute('aria-pressed', 'false');
+ });
b.classList.add('is-active');
+ b.setAttribute('aria-pressed', 'true');
if (sliderInput) {
sliderInput.max = String(entries.length);
var n = Math.min(parseInt(sliderInput.value, 10), entries.length);
@@ -147,7 +161,7 @@
var dc = defaultCount();
var slider = document.createElement('div');
slider.className = 'rv-word-slider';
- slider.innerHTML = '
';
+ }
+
+ public function render(PhpRenderer $view, SitePageBlockRepresentation $block,
+ $templateViewScript = 'common/block-layout/semantic-map')
+ {
+ return $view->partial($templateViewScript, ['block' => $block]);
+ }
+}
diff --git a/src/Site/ResourcePageBlockLayout/SimilarItems.php b/src/Site/ResourcePageBlockLayout/SimilarItems.php
new file mode 100644
index 00000000..e6e21c9f
--- /dev/null
+++ b/src/Site/ResourcePageBlockLayout/SimilarItems.php
@@ -0,0 +1,26 @@
+partial('common/resource-page-block-layout/similar-items', [
+ 'resource' => $resource,
+ ]);
+ }
+}
diff --git a/src/View/Helper/DashboardAssets.php b/src/View/Helper/DashboardAssets.php
index d9e4cc72..8008e0d1 100644
--- a/src/View/Helper/DashboardAssets.php
+++ b/src/View/Helper/DashboardAssets.php
@@ -156,6 +156,7 @@ class DashboardAssets extends AbstractHelper
'compare' => ['js/dashboard-compare-unify.js', 'js/dashboard-compare.js'],
'network' => ['js/dashboard-network-explorer.js'],
'whatsNew' => ['js/dashboard-whats-new.js'],
+ 'semanticMap' => ['js/semantic-map.js'],
];
/**
@@ -273,6 +274,21 @@ public function __invoke(array $options = [])
return $this;
}
+ // Semantic Map block: one ECharts scatter and its own controller. Keep
+ // the word-cloud, MapLibre, d3-force and dashboard builder bundle out of
+ // this page; the controller lazy-loads ECharts when the block nears view.
+ if (!empty($options['semanticMap'])) {
+ $headLink->appendStylesheet($asset('css/dre-visualizations.css'));
+ $headScript->appendScript('window.RV_LIBS=Object.assign(' . json_encode([
+ 'echarts' => $asset(self::ECHARTS_JS),
+ ], JSON_UNESCAPED_SLASHES) . ', window.RV_LIBS||{});');
+ $headScript->appendFile($asset('js/dashboard-core.js'), 'text/javascript', $defer);
+ foreach (self::CONTROLLERS['semanticMap'] as $script) {
+ $headScript->appendFile($asset($script), 'text/javascript', $defer);
+ }
+ return $this;
+ }
+
if ($cdn) {
$headLink->appendStylesheet($asset('css/dre-visualizations.css'));
if ($controller === 'dashboard') {
diff --git a/tests/AggregatorsTest.php b/tests/AggregatorsTest.php
index 7312e3bd..cee6fe9b 100644
--- a/tests/AggregatorsTest.php
+++ b/tests/AggregatorsTest.php
@@ -398,6 +398,30 @@ function check(bool $cond, string $msg): void
check(($roleOf['A'] ?? null) === 'author', 'coAuthorNetwork: A role is author');
check(($roleOf['C'] ?? null) === 'both', 'coAuthorNetwork: C role is both (author and editor)');
+// --- media charts: duration bands + curated PHP word-form fallback ---
+$duration = A::buildDurationHistogram([0, 15 * 60, 25 * 60, 65 * 60]);
+check($duration === [
+ ['name' => 'Under 20 min', 'value' => 1],
+ ['name' => '20–30 min', 'value' => 1],
+ ['name' => '60 min +', 'value' => 1],
+], 'duration histogram drops invalid/empty bands and preserves natural order');
+$fallbackCloud = A::buildTranscriptWordCloud([
+ '[music] Speaker 1: Studies studying study. Work working works.',
+ 'Études étude. Travaux travail. Communities community.',
+]);
+$fallbackCounts = [];
+foreach ($fallbackCloud ?? [] as $entry) {
+ $fallbackCounts[$entry['name']] = $entry['value'];
+}
+check(($fallbackCounts['study'] ?? null) === 3
+ && ($fallbackCounts['work'] ?? null) === 3
+ && ($fallbackCounts['étude'] ?? null) === 2
+ && ($fallbackCounts['travail'] ?? null) === 2
+ && ($fallbackCounts['community'] ?? null) === 2,
+ 'transcript fallback merges curated English/French word forms into readable lemmas');
+check(!isset($fallbackCounts['speaker'], $fallbackCounts['music']),
+ 'transcript fallback strips diarisation labels and audio cues');
+
// --- knowledge graph (IDF-ranked shared-item discovery) ---
$kgItems = [
1 => ['title' => 'Center', 'class_label' => 'Article', 'class_term' => 'fabio:JournalArticle', 'template_id' => 11],
diff --git a/tests/AmiraProfileTest.php b/tests/AmiraProfileTest.php
index fa691f0c..7908a77b 100644
--- a/tests/AmiraProfileTest.php
+++ b/tests/AmiraProfileTest.php
@@ -32,6 +32,18 @@ function profileCheck(bool $condition, string $message): void
'itemSet' => $profile->itemSet('podcasts'),
'field' => 'bibo:content',
], 'word-cloud corpora resolve the shared item-set key');
+$embeddingCorpora = $profile->embeddingCorpora();
+profileCheck(count($embeddingCorpora) === 6
+ && ($embeddingCorpora[0] ?? null) === [
+ 'id' => 'podcasts',
+ 'label' => 'Podcast',
+ 'selector' => 'itemSet',
+ 'selectorId' => $profile->itemSet('podcasts'),
+ 'textFields' => ['dcterms:abstract', 'bibo:content'],
+ ]
+ && ($embeddingCorpora[5]['selector'] ?? null) === 'template'
+ && ($embeddingCorpora[5]['selectorId'] ?? null) === $profile->template('researchItems'),
+ 'six embedding corpora resolve item-set and resource-template selectors');
profileCheck(count($profile->featuredCollections()) === 6,
'featured collections are loaded from the validated installation profile');
profileCheck(($profile->universityLabels()['University Joseph Ki-Zerbo'] ?? null) === 'Université Joseph Ki-Zerbo',
@@ -51,6 +63,9 @@ function profileCheck(bool $condition, string $message): void
'non-positive required id' => static function (array &$data): void { $data['itemSets']['youtube'] = 0; },
'unknown corpus item-set key' => static function (array &$data): void { $data['wordcloudCorpora'][0]['itemSetKey'] = 'missing'; },
'duplicate corpus id' => static function (array &$data): void { $data['wordcloudCorpora'][1]['id'] = $data['wordcloudCorpora'][0]['id']; },
+ 'embedding corpus with both selectors' => static function (array &$data): void { $data['embeddingCorpora'][0]['templateKey'] = 'projects'; },
+ 'embedding corpus with no text fields' => static function (array &$data): void { $data['embeddingCorpora'][0]['textFields'] = []; },
+ 'duplicate embedding corpus id' => static function (array &$data): void { $data['embeddingCorpora'][1]['id'] = $data['embeddingCorpora'][0]['id']; },
'duplicate featured collection slug' => static function (array &$data): void { $data['featuredCollections'][1]['slug'] = $data['featuredCollections'][0]['slug']; },
'unknown featured item-set key' => static function (array &$data): void { $data['featuredCollections'][0]['itemSetKey'] = 'missing'; },
];
diff --git a/tests/browser/semantic-fixture.html b/tests/browser/semantic-fixture.html
new file mode 100644
index 00000000..2b7eeac0
--- /dev/null
+++ b/tests/browser/semantic-fixture.html
@@ -0,0 +1,78 @@
+
+
+
+
+
+ DRE semantic components fixture
+
+
+
+
+
+
+
+
+
+
+
+
+ Explore records with similar subjects and descriptions, even when their language or resource type differs.
+
+
+
+
+
diff --git a/tools/embeddings/build_embeddings.py b/tools/embeddings/build_embeddings.py
new file mode 100644
index 00000000..db6e58ba
--- /dev/null
+++ b/tools/embeddings/build_embeddings.py
@@ -0,0 +1,937 @@
+#!/usr/bin/env python3
+"""Build AMIRA's shared semantic map, recommendations, and vector release.
+
+The source is the unauthenticated public Omeka API. Six profile-declared
+corpora are converted to comparable, length-bounded archival cards, embedded
+in one multilingual Gemini space, then projected with UMAP. Full vectors stay
+in a gitignored incremental cache and release bundle; only compact derived JSON
+is committed under ``asset/data/embeddings``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import random
+import re
+import sys
+import time
+from array import array
+from collections import Counter
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable, Iterable, Sequence
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+PROFILE_PATH = REPO_ROOT / "config" / "amira-profile.json"
+OUT_DIR = REPO_ROOT / "asset" / "data" / "embeddings"
+CACHE_PATH = OUT_DIR / "cache.json"
+RELEASE_DIR = OUT_DIR / "release"
+
+API_BASE = os.environ.get(
+ "OMEKA_API_BASE", "https://data.africamultiple.uni-bayreuth.de"
+).rstrip("/")
+MODEL = os.environ.get("GEMINI_EMBEDDING_MODEL", "gemini-embedding-2")
+OUTPUT_DIMS = int(os.environ.get("GEMINI_EMBEDDING_DIMS", "768"))
+TASK_PREFIX = os.environ.get(
+ "GEMINI_EMBEDDING_TASK_PREFIX", "task: sentence similarity | query: "
+)
+BATCH_SIZE = int(os.environ.get("GEMINI_EMBEDDING_BATCH", "32"))
+INTER_BATCH_DELAY_S = float(os.environ.get("GEMINI_EMBEDDING_DELAY_S", "0.5"))
+FLUSH_EVERY_BATCHES = int(os.environ.get("GEMINI_EMBEDDING_FLUSH_EVERY", "5"))
+
+CARD_MAX_WORDS = 900
+LOW_SIGNAL_THRESHOLD = 100
+SIMILAR_TOP_K = 12
+UMAP_NEIGHBORS = 15
+UMAP_MIN_DIST = 0.1
+UMAP_SEED = 42
+SCHEMA_VERSION = 1
+
+TERM_RE = re.compile(r"^[a-z][a-z0-9]*:[A-Za-z][A-Za-z0-9]*$")
+ID_RE = re.compile(r"^[a-z][a-z0-9-]*$")
+TAG_RE = re.compile(r"<[^>]+>")
+SPACE_RE = re.compile(r"\s+")
+
+
+@dataclass(frozen=True)
+class CorpusConfig:
+ id: str
+ label: str
+ selector: str
+ selector_id: int
+ text_fields: tuple[str, ...]
+
+
+@dataclass
+class Card:
+ id: int
+ corpus: str
+ type_label: str
+ title: str
+ text: str
+ content_hash: str
+ input_chars: int
+ input_words: int
+ truncated: bool
+ low_signal: bool
+ languages: list[str]
+
+
+def utc_now() -> str:
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def compact_json(path: Path, payload: Any) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ tmp.write_text(
+ json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n",
+ encoding="utf-8",
+ )
+ tmp.replace(path)
+
+
+def pretty_json(path: Path, payload: Any) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ tmp.write_text(
+ json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+ )
+ tmp.replace(path)
+
+
+def load_profile(path: Path = PROFILE_PATH) -> tuple[dict[str, Any], list[CorpusConfig]]:
+ profile = json.loads(path.read_text(encoding="utf-8"))
+ item_sets = profile.get("itemSets")
+ templates = profile.get("templates")
+ raw_corpora = profile.get("embeddingCorpora")
+ if not isinstance(item_sets, dict) or not isinstance(templates, dict):
+ raise ValueError("AMIRA profile is missing item-set/template selectors")
+ if not isinstance(raw_corpora, list) or not raw_corpora:
+ raise ValueError("AMIRA profile has no embeddingCorpora")
+
+ corpora: list[CorpusConfig] = []
+ seen: set[str] = set()
+ for raw in raw_corpora:
+ if not isinstance(raw, dict):
+ raise ValueError("embeddingCorpora entries must be objects")
+ corpus_id = raw.get("id")
+ label = raw.get("label")
+ item_set_key = raw.get("itemSetKey")
+ template_key = raw.get("templateKey")
+ fields = raw.get("textFields")
+ has_item_set = isinstance(item_set_key, str) and item_set_key in item_sets
+ has_template = isinstance(template_key, str) and template_key in templates
+ if (
+ not isinstance(corpus_id, str)
+ or not ID_RE.fullmatch(corpus_id)
+ or corpus_id in seen
+ or not isinstance(label, str)
+ or not label.strip()
+ or has_item_set == has_template
+ or not isinstance(fields, list)
+ or not fields
+ or any(not isinstance(field, str) or not TERM_RE.fullmatch(field) for field in fields)
+ ):
+ raise ValueError(f"Invalid embedding corpus configuration: {raw!r}")
+ selector = "item_set_id" if has_item_set else "resource_template_id"
+ selector_id = item_sets[item_set_key] if has_item_set else templates[template_key]
+ if not isinstance(selector_id, int) or selector_id < 1:
+ raise ValueError(f"Invalid embedding selector id: {raw!r}")
+ corpora.append(
+ CorpusConfig(
+ id=corpus_id,
+ label=label.strip(),
+ selector=selector,
+ selector_id=selector_id,
+ text_fields=tuple(fields),
+ )
+ )
+ seen.add(corpus_id)
+ return profile, corpora
+
+
+def fetch_json(url: str, attempts: int = 4) -> Any:
+ for attempt in range(1, attempts + 1):
+ try:
+ request = Request(url, headers={"User-Agent": "dre-embeddings/1.0"})
+ with urlopen(request, timeout=90) as response:
+ return json.load(response)
+ except Exception:
+ if attempt == attempts:
+ raise
+ time.sleep(3 * attempt)
+ raise AssertionError("unreachable")
+
+
+def fetch_corpus(corpus: CorpusConfig, fetcher: Callable[[str], Any] = fetch_json) -> list[dict]:
+ items: list[dict] = []
+ page = 1
+ while True:
+ query = urlencode({corpus.selector: corpus.selector_id, "per_page": 100, "page": page})
+ batch = fetcher(f"{API_BASE}/api/items?{query}")
+ if not isinstance(batch, list):
+ raise ValueError(f"Omeka returned a non-list for {corpus.id}")
+ items.extend(item for item in batch if isinstance(item, dict))
+ if len(batch) < 100:
+ return items
+ page += 1
+ time.sleep(0.15)
+
+
+def value_text(value: Any) -> str:
+ if not isinstance(value, dict):
+ return ""
+ for key in ("display_title", "@value", "@id"):
+ candidate = value.get(key)
+ if isinstance(candidate, str) and candidate.strip():
+ return candidate.strip()
+ return ""
+
+
+def values(item: dict, term: str) -> list[str]:
+ raw = item.get(term)
+ if not isinstance(raw, list):
+ return []
+ out: list[str] = []
+ seen: set[str] = set()
+ for value in raw:
+ text = value_text(value)
+ if text and text not in seen:
+ out.append(text)
+ seen.add(text)
+ return out
+
+
+def linked_ids(item: dict, term: str) -> list[int]:
+ out: list[int] = []
+ for value in item.get(term) or []:
+ if not isinstance(value, dict):
+ continue
+ item_id = value.get("value_resource_id")
+ if isinstance(item_id, int) and item_id > 0 and item_id not in out:
+ out.append(item_id)
+ return out
+
+
+def clean_text(text: str) -> str:
+ return SPACE_RE.sub(" ", TAG_RE.sub(" ", text)).strip()
+
+
+def title_of(item: dict) -> str:
+ title = item.get("o:title")
+ if isinstance(title, str) and title.strip():
+ return clean_text(title)
+ titles = values(item, "dcterms:title")
+ return clean_text(titles[0]) if titles else f"Item {item.get('o:id', '')}".strip()
+
+
+def relationship_context(
+ item: dict,
+ item_id: int,
+ project_items: dict[int, dict],
+ section_items: dict[int, dict],
+) -> tuple[list[str], list[str]]:
+ project_ids: list[int] = []
+ section_ids: list[int] = []
+ for term in ("dcterms:isPartOf", "dcterms:relation"):
+ for linked_id in linked_ids(item, term):
+ if linked_id in project_items and linked_id not in project_ids:
+ project_ids.append(linked_id)
+ if linked_id in section_items and linked_id not in section_ids:
+ section_ids.append(linked_id)
+ if item_id in project_items:
+ project_ids = [item_id]
+ for project_id in project_ids:
+ for linked_id in linked_ids(project_items[project_id], "dcterms:isPartOf"):
+ if linked_id in section_items and linked_id not in section_ids:
+ section_ids.append(linked_id)
+ return (
+ [title_of(project_items[value]) for value in project_ids],
+ [title_of(section_items[value]) for value in section_ids],
+ )
+
+
+def build_card(
+ item: dict,
+ corpus: CorpusConfig,
+ project_items: dict[int, dict],
+ section_items: dict[int, dict],
+ max_words: int = CARD_MAX_WORDS,
+ low_signal_threshold: int = LOW_SIGNAL_THRESHOLD,
+) -> Card:
+ item_id = item.get("o:id")
+ if not isinstance(item_id, int) or item_id < 1:
+ raise ValueError("Public Omeka item has no positive integer o:id")
+ title = title_of(item)
+ projects, sections = relationship_context(item, item_id, project_items, section_items)
+ subjects = values(item, "dcterms:subject")
+ places = values(item, "dcterms:spatial")
+ languages = values(item, "dcterms:language")
+
+ parts = [f"Title: {title}", f"Type: {corpus.label}"]
+ if projects:
+ parts.append("Project: " + "; ".join(projects))
+ if sections:
+ parts.append("Research section: " + "; ".join(sections))
+ if subjects:
+ parts.append("Subjects: " + "; ".join(subjects))
+ if places:
+ parts.append("Places: " + "; ".join(places))
+ if languages:
+ parts.append("Languages: " + "; ".join(languages))
+
+ narrative = ""
+ narrative_field = ""
+ for field in corpus.text_fields:
+ candidate = clean_text(" ".join(values(item, field)))
+ if candidate:
+ narrative = candidate
+ narrative_field = field
+ break
+ if narrative:
+ label = "Abstract" if "abstract" in narrative_field.lower() else "Description"
+ parts.append(f"{label}: {narrative}")
+
+ full_text = "\n".join(parts).strip()
+ words = full_text.split()
+ truncated = len(words) > max_words
+ if truncated:
+ full_text = " ".join(words[:max_words])
+ input_chars = len(full_text)
+ input_words = len(full_text.split())
+ return Card(
+ id=item_id,
+ corpus=corpus.id,
+ type_label=corpus.label,
+ title=title,
+ text=full_text,
+ content_hash=hashlib.sha256(full_text.encode("utf-8")).hexdigest(),
+ input_chars=input_chars,
+ input_words=input_words,
+ truncated=truncated,
+ low_signal=input_chars < low_signal_threshold,
+ languages=languages,
+ )
+
+
+def load_cards(
+ corpora: Sequence[CorpusConfig],
+ fetcher: Callable[[str], Any] = fetch_json,
+) -> tuple[list[Card], dict[str, dict[str, int]]]:
+ by_corpus: dict[str, list[dict]] = {}
+ counts: dict[str, dict[str, int]] = {}
+ for corpus in corpora:
+ raw = fetch_corpus(corpus, fetcher)
+ public = [item for item in raw if item.get("o:is_public") is True]
+ by_corpus[corpus.id] = public
+ counts[corpus.id] = {
+ "fetched": len(raw),
+ "public": len(public),
+ "nonPublicSkipped": len(raw) - len(public),
+ }
+ print(f" {corpus.id}: {len(public)} public item(s)")
+
+ project_items = {
+ item["o:id"]: item
+ for item in by_corpus.get("projects", [])
+ if isinstance(item.get("o:id"), int)
+ }
+ section_items = {
+ item["o:id"]: item
+ for item in by_corpus.get("sections", [])
+ if isinstance(item.get("o:id"), int)
+ }
+
+ cards: list[Card] = []
+ seen_ids: set[int] = set()
+ for corpus in corpora:
+ for item in by_corpus[corpus.id]:
+ card = build_card(item, corpus, project_items, section_items)
+ if card.id in seen_ids:
+ raise ValueError(f"Item {card.id} appears in more than one embedding corpus")
+ seen_ids.add(card.id)
+ cards.append(card)
+ corpus_cards = [card for card in cards if card.corpus == corpus.id]
+ counts[corpus.id].update(
+ {
+ "cards": len(corpus_cards),
+ "lowSignal": sum(card.low_signal for card in corpus_cards),
+ "truncated": sum(card.truncated for card in corpus_cards),
+ }
+ )
+ cards.sort(key=lambda card: card.id)
+ return cards, counts
+
+
+def empty_cache() -> dict[str, Any]:
+ return {
+ "schemaVersion": SCHEMA_VERSION,
+ "model": MODEL,
+ "dims": OUTPUT_DIMS,
+ "taskPrefix": TASK_PREFIX,
+ "items": {},
+ }
+
+
+def load_cache(path: Path = CACHE_PATH) -> dict[str, Any]:
+ if not path.is_file():
+ return empty_cache()
+ try:
+ cache = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError):
+ return empty_cache()
+ compatible = (
+ cache.get("schemaVersion") == SCHEMA_VERSION
+ and cache.get("model") == MODEL
+ and cache.get("dims") == OUTPUT_DIMS
+ and cache.get("taskPrefix") == TASK_PREFIX
+ and isinstance(cache.get("items"), dict)
+ )
+ return cache if compatible else empty_cache()
+
+
+class GeminiEmbedder:
+ def __init__(self, api_key: str):
+ from google import genai
+
+ self._client = genai.Client(api_key=api_key)
+
+ def close(self) -> None:
+ self._client.close()
+
+ def __call__(self, texts: Sequence[str], retries: int = 5) -> list[list[float]]:
+ from google.genai import types
+ from google.genai.errors import APIError
+
+ contents = [
+ types.Content(
+ role="user",
+ parts=[types.Part.from_text(text=TASK_PREFIX + text)],
+ )
+ for text in texts
+ ]
+ delay = 2.0
+ for attempt in range(retries):
+ try:
+ response = self._client.models.embed_content(
+ model=MODEL,
+ contents=contents,
+ config=types.EmbedContentConfig(
+ output_dimensionality=OUTPUT_DIMS,
+ auto_truncate=False,
+ ),
+ )
+ vectors = [list(embedding.values or []) for embedding in response.embeddings or []]
+ if len(vectors) != len(texts):
+ raise RuntimeError(
+ f"Gemini returned {len(vectors)} vectors for {len(texts)} Content objects"
+ )
+ if any(len(vector) != OUTPUT_DIMS for vector in vectors):
+ raise RuntimeError("Gemini returned an unexpected embedding dimension")
+ return vectors
+ except APIError as error:
+ status = getattr(error, "status_code", None) or getattr(error, "code", None)
+ if status not in (429, 500, 502, 503, 504) or attempt == retries - 1:
+ raise
+ time.sleep(delay + random.random())
+ delay = min(delay * 2, 60)
+ raise AssertionError("unreachable")
+
+
+def update_cache(
+ cards: Sequence[Card],
+ cache: dict[str, Any],
+ embed: Callable[[Sequence[str]], list[list[float]]],
+ scope: str,
+ cache_path: Path = CACHE_PATH,
+) -> dict[str, int]:
+ items: dict[str, Any] = cache["items"]
+ live_ids = {str(card.id) for card in cards}
+ stale = [item_id for item_id in items if item_id not in live_ids]
+ for item_id in stale:
+ del items[item_id]
+
+ pending = [
+ card
+ for card in cards
+ if scope == "all"
+ or str(card.id) not in items
+ or items[str(card.id)].get("hash") != card.content_hash
+ ]
+ reused = len(cards) - len(pending)
+ batches_since_flush = 0
+ try:
+ for offset in range(0, len(pending), BATCH_SIZE):
+ batch = pending[offset : offset + BATCH_SIZE]
+ started = time.time()
+ vectors = embed([card.text for card in batch])
+ if len(vectors) != len(batch):
+ raise RuntimeError("Embedding provider returned the wrong batch size")
+ for card, vector in zip(batch, vectors):
+ if len(vector) != OUTPUT_DIMS:
+ raise RuntimeError(f"Item {card.id} has {len(vector)} dimensions, expected {OUTPUT_DIMS}")
+ items[str(card.id)] = {
+ "hash": card.content_hash,
+ "vector": vector,
+ }
+ batches_since_flush += 1
+ print(f" embedded {min(offset + len(batch), len(pending))}/{len(pending)}")
+ if batches_since_flush >= FLUSH_EVERY_BATCHES:
+ compact_json(cache_path, cache)
+ batches_since_flush = 0
+ elapsed = time.time() - started
+ if INTER_BATCH_DELAY_S > elapsed and offset + BATCH_SIZE < len(pending):
+ time.sleep(INTER_BATCH_DELAY_S - elapsed)
+ except BaseException:
+ # Preserve paid-for successful batches across API failures and Ctrl-C.
+ compact_json(cache_path, cache)
+ raise
+
+ compact_json(cache_path, cache)
+ return {"embedded": len(pending), "reused": reused, "staleRemoved": len(stale)}
+
+
+def vector_matrix(cards: Sequence[Card], cache: dict[str, Any]):
+ import numpy as np
+
+ matrix = np.asarray(
+ [cache["items"][str(card.id)]["vector"] for card in cards], dtype=np.float32
+ )
+ if matrix.shape != (len(cards), OUTPUT_DIMS):
+ raise ValueError(f"Unexpected embedding matrix shape: {matrix.shape}")
+ norms = np.linalg.norm(matrix, axis=1, keepdims=True)
+ if np.any(norms == 0):
+ raise ValueError("Embedding matrix contains a zero vector")
+ return matrix / norms
+
+
+def project_and_cluster(matrix):
+ import numpy as np
+ import umap
+ from sklearn.cluster import KMeans
+
+ count = matrix.shape[0]
+ if count == 1:
+ return np.zeros((1, 2), dtype=np.float32), np.zeros(1, dtype=np.int32), 1
+ if count == 2:
+ # UMAP requires at least two neighbours. Keep the tiny-corpus contract
+ # valid and deterministic without changing the normal projection path.
+ coords = np.asarray([[-1.0, 0.0], [1.0, 0.0]], dtype=np.float32)
+ return coords, np.asarray([0, 1], dtype=np.int32), 2
+ reducer = umap.UMAP(
+ n_components=2,
+ n_neighbors=min(UMAP_NEIGHBORS, count - 1),
+ min_dist=UMAP_MIN_DIST,
+ metric="cosine",
+ random_state=UMAP_SEED,
+ n_jobs=1,
+ )
+ coords = reducer.fit_transform(matrix)
+ cluster_count = min(count, max(2, min(30, round(math.sqrt(count / 2)))))
+ clusters = KMeans(n_clusters=cluster_count, random_state=UMAP_SEED, n_init="auto").fit_predict(matrix)
+ return coords, clusters, cluster_count
+
+
+def similar_items(cards: Sequence[Card], matrix, top_k: int = SIMILAR_TOP_K) -> dict[str, list[dict[str, Any]]]:
+ import numpy as np
+
+ eligible = [index for index, card in enumerate(cards) if not card.low_signal]
+ if len(eligible) < 2:
+ return {}
+ eligible_matrix = matrix[eligible]
+ output: dict[str, list[dict[str, Any]]] = {}
+ k = min(top_k, len(eligible) - 1)
+ for start in range(0, len(eligible), 256):
+ source_indices = eligible[start : start + 256]
+ scores = matrix[source_indices] @ eligible_matrix.T
+ for row_index, source_index in enumerate(source_indices):
+ source_position = start + row_index
+ scores[row_index, source_position] = -np.inf
+ candidate_positions = np.argpartition(scores[row_index], -k)[-k:]
+ candidate_positions = candidate_positions[
+ np.argsort(-scores[row_index, candidate_positions])
+ ]
+ output[str(cards[source_index].id)] = [
+ {
+ "id": cards[eligible[position]].id,
+ "score": round(float(scores[row_index, position]), 6),
+ }
+ for position in candidate_positions
+ if math.isfinite(float(scores[row_index, position]))
+ and float(scores[row_index, position]) > 0
+ ]
+ return output
+
+
+def catalog_entry(card: Card) -> dict[str, Any]:
+ entry: dict[str, Any] = {
+ "title": card.title,
+ "type": card.corpus,
+ "typeLabel": card.type_label,
+ "lowSignal": card.low_signal,
+ }
+ if card.languages:
+ entry["languages"] = card.languages
+ return entry
+
+
+def recommendation_quality(
+ cards: Sequence[Card], recommendations: dict[str, list[dict[str, Any]]]
+) -> dict[str, Any]:
+ by_id = {str(card.id): card for card in cards}
+ cross_type = 0
+ cross_language = 0
+ examples: list[dict[str, Any]] = []
+ scores: list[float] = []
+ for source_id, neighbours in recommendations.items():
+ source = by_id[source_id]
+ if any(by_id[str(n["id"])].corpus != source.corpus for n in neighbours):
+ cross_type += 1
+ source_langs = set(source.languages)
+ language_match = next(
+ (
+ n
+ for n in neighbours
+ if source_langs
+ and set(by_id[str(n["id"])].languages)
+ and source_langs.isdisjoint(by_id[str(n["id"])].languages)
+ ),
+ None,
+ )
+ if language_match:
+ cross_language += 1
+ if len(examples) < 12:
+ target = by_id[str(language_match["id"])]
+ examples.append(
+ {
+ "source": source.id,
+ "sourceLanguages": source.languages,
+ "target": target.id,
+ "targetLanguages": target.languages,
+ "score": language_match["score"],
+ }
+ )
+ scores.extend(float(neighbour["score"]) for neighbour in neighbours)
+ return {
+ "sources": len(recommendations),
+ "sourcesWithCrossTypeNeighbour": cross_type,
+ "sourcesWithCrossLanguageNeighbour": cross_language,
+ "meanSimilarity": round(sum(scores) / len(scores), 6) if scores else None,
+ "crossLanguageExamples": examples,
+ }
+
+
+def write_release(matrix, cards: Sequence[Card], generated_at: str) -> dict[str, Any]:
+ RELEASE_DIR.mkdir(parents=True, exist_ok=True)
+ vector_path = RELEASE_DIR / "vectors.f32"
+ payload = array("f", matrix.astype(" dict[str, Any]:
+ generated_at = utc_now()
+ counts = Counter(card.corpus for card in cards)
+ map_payload = {
+ "schemaVersion": SCHEMA_VERSION,
+ "model": MODEL,
+ "dimensions": OUTPUT_DIMS,
+ "generatedAt": generated_at,
+ "lowSignalThreshold": LOW_SIGNAL_THRESHOLD,
+ "cardMaxWords": CARD_MAX_WORDS,
+ "umap": {
+ "neighbors": UMAP_NEIGHBORS,
+ "minDist": UMAP_MIN_DIST,
+ "metric": "cosine",
+ "seed": UMAP_SEED,
+ },
+ "clustering": {"algorithm": "k-means", "clusters": cluster_count, "seed": UMAP_SEED},
+ "types": dict(sorted(counts.items())),
+ "items": [
+ {
+ "id": card.id,
+ "x": round(float(coords[index][0]), 6),
+ "y": round(float(coords[index][1]), 6),
+ "type": card.corpus,
+ "typeLabel": card.type_label,
+ "cluster": int(clusters[index]),
+ "lowSignal": card.low_signal,
+ "title": card.title,
+ }
+ for index, card in enumerate(cards)
+ ],
+ }
+ map_path = OUT_DIR / "map.json"
+ similar_path = OUT_DIR / "similar.json"
+ compact_json(map_path, map_payload)
+ compact_json(
+ similar_path,
+ {
+ "schemaVersion": SCHEMA_VERSION,
+ "model": MODEL,
+ "dimensions": OUTPUT_DIMS,
+ "generatedAt": generated_at,
+ "topK": SIMILAR_TOP_K,
+ "catalog": {str(card.id): catalog_entry(card) for card in cards},
+ "items": recommendations,
+ },
+ )
+ quality = recommendation_quality(cards, recommendations)
+ release_manifest = write_release(matrix, cards, generated_at)
+ report = {
+ "schemaVersion": SCHEMA_VERSION,
+ "generatedAt": generated_at,
+ "source": API_BASE,
+ "model": MODEL,
+ "dimensions": OUTPUT_DIMS,
+ "taskPrefix": TASK_PREFIX,
+ "cardPolicy": {
+ "maxWords": CARD_MAX_WORDS,
+ "lowSignalThresholdChars": LOW_SIGNAL_THRESHOLD,
+ "allItemsShareTheSameLengthBudget": True,
+ },
+ "corpora": corpus_counts,
+ "totals": {
+ "cards": len(cards),
+ "lowSignal": sum(card.low_signal for card in cards),
+ "recommendable": sum(not card.low_signal for card in cards),
+ "truncated": sum(card.truncated for card in cards),
+ "maximumInputWords": max((card.input_words for card in cards), default=0),
+ },
+ "cache": cache_stats,
+ "artifactBytes": {
+ "map": map_path.stat().st_size,
+ "similar": similar_path.stat().st_size,
+ },
+ "recommendations": quality,
+ "release": release_manifest,
+ }
+ pretty_json(OUT_DIR / "report.json", report)
+ return report
+
+
+def validate_artifacts(out_dir: Path = OUT_DIR, release_dir: Path = RELEASE_DIR) -> dict[str, int]:
+ """Fail closed when derived JSON and the vector release disagree."""
+ paths = {
+ "map": out_dir / "map.json",
+ "similar": out_dir / "similar.json",
+ "report": out_dir / "report.json",
+ "ids": release_dir / "ids.json",
+ "manifest": release_dir / "manifest.json",
+ "vectors": release_dir / "vectors.f32",
+ }
+ missing = [str(path) for path in paths.values() if not path.is_file()]
+ if missing:
+ raise ValueError(f"Missing semantic artifacts: {', '.join(missing)}")
+
+ try:
+ map_data = json.loads(paths["map"].read_text(encoding="utf-8"))
+ similar_data = json.loads(paths["similar"].read_text(encoding="utf-8"))
+ report = json.loads(paths["report"].read_text(encoding="utf-8"))
+ release_ids = json.loads(paths["ids"].read_text(encoding="utf-8"))
+ manifest = json.loads(paths["manifest"].read_text(encoding="utf-8"))
+ except (json.JSONDecodeError, OSError) as exc:
+ raise ValueError(f"Unreadable semantic artifact: {exc}") from exc
+
+ metadata = (map_data, similar_data, report, manifest)
+ if any(payload.get("schemaVersion") != SCHEMA_VERSION for payload in metadata):
+ raise ValueError("Semantic artifacts do not share the expected schema version.")
+ if any(payload.get("model") != MODEL for payload in (map_data, similar_data, report, manifest)):
+ raise ValueError("Semantic artifacts do not share the configured embedding model.")
+ if any(payload.get("dimensions") != OUTPUT_DIMS for payload in (map_data, similar_data, report, manifest)):
+ raise ValueError("Semantic artifacts do not share the configured vector dimensions.")
+
+ map_items = map_data.get("items")
+ catalog = similar_data.get("catalog")
+ recommendations = similar_data.get("items")
+ if not isinstance(map_items, list) or not isinstance(catalog, dict) or not isinstance(recommendations, dict):
+ raise ValueError("Semantic map or recommendation payload has an invalid shape.")
+ if not isinstance(release_ids, list):
+ raise ValueError("Release ids.json must contain a list.")
+
+ map_ids = [str(row.get("id")) for row in map_items]
+ catalog_ids = list(catalog)
+ vector_ids = [str(row.get("id")) for row in release_ids]
+ if len(map_ids) != len(set(map_ids)) or len(vector_ids) != len(set(vector_ids)):
+ raise ValueError("Semantic artifacts contain duplicate resource IDs.")
+ if set(map_ids) != set(catalog_ids) or map_ids != vector_ids:
+ raise ValueError("Map, recommendation catalog, and vector row IDs disagree.")
+
+ low_signal_ids = {str(row["id"]) for row in map_items if row.get("lowSignal") is True}
+ for source_id, neighbours in recommendations.items():
+ if source_id not in catalog or source_id in low_signal_ids or not isinstance(neighbours, list):
+ raise ValueError(f"Invalid recommendation source: {source_id}")
+ for neighbour in neighbours:
+ target_id = str(neighbour.get("id"))
+ score = neighbour.get("score")
+ if (
+ target_id not in catalog
+ or target_id == source_id
+ or target_id in low_signal_ids
+ or not isinstance(score, (int, float))
+ or not math.isfinite(score)
+ or score <= 0
+ ):
+ raise ValueError(f"Invalid recommendation {source_id} -> {target_id}")
+
+ vector_bytes = paths["vectors"].read_bytes()
+ expected_bytes = len(vector_ids) * OUTPUT_DIMS * 4
+ if len(vector_bytes) != expected_bytes:
+ raise ValueError(f"Vector byte length is {len(vector_bytes)}; expected {expected_bytes}.")
+ if manifest.get("count") != len(vector_ids):
+ raise ValueError("Vector manifest count does not match ids.json.")
+ if manifest.get("vectorsSha256") != hashlib.sha256(vector_bytes).hexdigest():
+ raise ValueError("Vector checksum does not match the release manifest.")
+
+ totals = report.get("totals", {})
+ if totals.get("cards") != len(map_ids) or totals.get("lowSignal") != len(low_signal_ids):
+ raise ValueError("Build report totals do not match the public artifact set.")
+ if report.get("artifactBytes") != {
+ "map": paths["map"].stat().st_size,
+ "similar": paths["similar"].stat().st_size,
+ }:
+ raise ValueError("Build report byte counts do not match the derived artifacts.")
+ return {
+ "cards": len(map_ids),
+ "recommendationSources": len(recommendations),
+ "lowSignal": len(low_signal_ids),
+ }
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ parser.add_argument("--scope", choices=("missing", "all"), default="missing")
+ parser.add_argument(
+ "--validate-profile",
+ action="store_true",
+ help="Validate and print corpus selectors without network or model access.",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Fetch public records and report card/cache counts without embedding.",
+ )
+ parser.add_argument(
+ "--validate-artifacts",
+ action="store_true",
+ help="Validate the complete derived and vector artifact set without network access.",
+ )
+ args = parser.parse_args()
+
+ if args.validate_artifacts:
+ summary = validate_artifacts()
+ print(
+ f"Validated {summary['cards']} semantic cards and "
+ f"{summary['recommendationSources']} recommendation sources."
+ )
+ return 0
+
+ _, corpora = load_profile()
+ if args.validate_profile:
+ for corpus in corpora:
+ print(f"{corpus.id}: {corpus.selector}={corpus.selector_id} ({', '.join(corpus.text_fields)})")
+ return 0
+
+ print(f"Fetching {len(corpora)} public corpora from {API_BASE}...")
+ cards, corpus_counts = load_cards(corpora)
+ if not cards:
+ print("No public cards were built.", file=sys.stderr)
+ return 1
+ cache = load_cache()
+ pending = sum(
+ args.scope == "all"
+ or str(card.id) not in cache["items"]
+ or cache["items"][str(card.id)].get("hash") != card.content_hash
+ for card in cards
+ )
+ print(
+ f"Built {len(cards)} cards: {sum(card.low_signal for card in cards)} low-signal, "
+ f"{sum(card.truncated for card in cards)} length-capped, {pending} to embed."
+ )
+ if args.dry_run:
+ return 0
+
+ api_key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+ if not api_key:
+ print("GEMINI_API_KEY (or GOOGLE_API_KEY) is required.", file=sys.stderr)
+ return 2
+
+ embedder = GeminiEmbedder(api_key)
+ try:
+ cache_stats = update_cache(cards, cache, embedder, args.scope)
+ finally:
+ embedder.close()
+ matrix = vector_matrix(cards, cache)
+ coords, clusters, cluster_count = project_and_cluster(matrix)
+ recommendations = similar_items(cards, matrix)
+ report = write_derived(
+ cards,
+ matrix,
+ coords,
+ clusters,
+ cluster_count,
+ recommendations,
+ corpus_counts,
+ cache_stats,
+ )
+ print(
+ f"Wrote map/similar/report for {report['totals']['cards']} public items; "
+ f"{report['recommendations']['sourcesWithCrossTypeNeighbour']} recommendation lists "
+ "contain another resource type."
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except KeyboardInterrupt:
+ print("Interrupted.", file=sys.stderr)
+ raise SystemExit(130)
diff --git a/tools/embeddings/requirements.txt b/tools/embeddings/requirements.txt
new file mode 100644
index 00000000..f6975514
--- /dev/null
+++ b/tools/embeddings/requirements.txt
@@ -0,0 +1,5 @@
+google-genai==2.16.0
+numpy==2.4.3
+numba==0.66.0
+scikit-learn==1.9.0
+umap-learn==0.5.12
diff --git a/tools/embeddings/tests/test_build_embeddings.py b/tools/embeddings/tests/test_build_embeddings.py
new file mode 100644
index 00000000..51fb5580
--- /dev/null
+++ b/tools/embeddings/tests/test_build_embeddings.py
@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / "build_embeddings.py"
+SPEC = importlib.util.spec_from_file_location("dre_build_embeddings", SCRIPT)
+assert SPEC and SPEC.loader
+emb = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = emb
+SPEC.loader.exec_module(emb)
+
+
+def value(text: str, item_id: int | None = None) -> dict:
+ row = {"type": "literal", "display_title": text}
+ if item_id is not None:
+ row.update({"type": "resource:item", "value_resource_id": item_id})
+ return row
+
+
+def item(item_id: int, title: str, *, public: bool = True, **properties) -> dict:
+ return {
+ "o:id": item_id,
+ "o:is_public": public,
+ "o:title": title,
+ **properties,
+ }
+
+
+class EmbeddingBuildTests(unittest.TestCase):
+ def test_repository_profile_resolves_six_mixed_selectors(self):
+ _, corpora = emb.load_profile()
+ self.assertEqual(6, len(corpora))
+ self.assertEqual(("podcasts", "item_set_id", 39095), (
+ corpora[0].id, corpora[0].selector, corpora[0].selector_id
+ ))
+ self.assertEqual(("research-items", "resource_template_id", 10), (
+ corpora[-1].id, corpora[-1].selector, corpora[-1].selector_id
+ ))
+
+ def test_value_precedence_and_public_fetch_pagination(self):
+ self.assertEqual("Linked title", emb.value_text({
+ "display_title": "Linked title", "@value": "Literal", "@id": "https://example.test"
+ }))
+ corpus = emb.CorpusConfig("projects", "Project", "resource_template_id", 5, ("dcterms:abstract",))
+ calls = []
+
+ def fetch(url: str):
+ calls.append(url)
+ return [item(i, f"Project {i}") for i in range(1, 101)] if len(calls) == 1 else []
+
+ rows = emb.fetch_corpus(corpus, fetch)
+ self.assertEqual(100, len(rows))
+ self.assertEqual(2, len(calls))
+ self.assertIn("resource_template_id=5", calls[0])
+
+ def test_card_uses_project_section_context_and_caps_long_text(self):
+ section = item(20, "Knowledges")
+ project = item(10, "Plural Archives", **{
+ "dcterms:isPartOf": [value("Knowledges", 20)]
+ })
+ research_item = item(1, "Archive recording", **{
+ "dcterms:isPartOf": [value("Plural Archives", 10)],
+ "dcterms:subject": [value("Oral history")],
+ "dcterms:spatial": [value("Ghana")],
+ "dcterms:language": [value("English")],
+ "dcterms:description": [{"@value": " ".join(f"word{i}" for i in range(50))}],
+ })
+ corpus = emb.CorpusConfig(
+ "research-items", "Research item", "resource_template_id", 10,
+ ("dcterms:description",),
+ )
+ card = emb.build_card(research_item, corpus, {10: project}, {20: section}, max_words=25)
+ self.assertIn("Project: Plural Archives", card.text)
+ self.assertIn("Research section: Knowledges", card.text)
+ self.assertTrue(card.truncated)
+ self.assertEqual(25, card.input_words)
+
+ def test_nonpublic_records_never_become_cards(self):
+ corpus = emb.CorpusConfig("projects", "Project", "resource_template_id", 5, ("dcterms:abstract",))
+
+ def fetch(url: str):
+ return [item(1, "Public"), item(2, "Private", public=False)]
+
+ # Keep the fake response to one page.
+ original = emb.fetch_corpus
+ try:
+ emb.fetch_corpus = lambda _corpus, _fetcher: fetch("")
+ cards, counts = emb.load_cards([corpus], fetch)
+ finally:
+ emb.fetch_corpus = original
+ self.assertEqual([1], [card.id for card in cards])
+ self.assertEqual(1, counts["projects"]["nonPublicSkipped"])
+
+ def test_incremental_cache_embeds_only_changed_cards_and_drops_stale(self):
+ old_dims, old_batch, old_delay = emb.OUTPUT_DIMS, emb.BATCH_SIZE, emb.INTER_BATCH_DELAY_S
+ emb.OUTPUT_DIMS, emb.BATCH_SIZE, emb.INTER_BATCH_DELAY_S = 3, 2, 0
+ try:
+ cards = [
+ emb.Card(1, "a", "A", "One", "one", "h1", 3, 1, False, True, []),
+ emb.Card(2, "b", "B", "Two", "two", "h2-new", 3, 1, False, False, []),
+ ]
+ cache = emb.empty_cache()
+ cache["items"] = {
+ "1": {"hash": "h1", "vector": [1.0, 0.0, 0.0]},
+ "2": {"hash": "h2-old", "vector": [0.0, 1.0, 0.0]},
+ "99": {"hash": "stale", "vector": [0.0, 0.0, 1.0]},
+ }
+ batches = []
+
+ def fake_embed(texts):
+ batches.append(list(texts))
+ return [[0.0, 1.0, 0.0] for _ in texts]
+
+ with tempfile.TemporaryDirectory() as directory:
+ stats = emb.update_cache(cards, cache, fake_embed, "missing", Path(directory) / "cache.json")
+ self.assertEqual([["two"]], batches)
+ self.assertEqual({"embedded": 1, "reused": 1, "staleRemoved": 1}, stats)
+ self.assertNotIn("99", cache["items"])
+ finally:
+ emb.OUTPUT_DIMS, emb.BATCH_SIZE, emb.INTER_BATCH_DELAY_S = old_dims, old_batch, old_delay
+
+ def test_similarity_excludes_low_signal_sources_and_neighbours(self):
+ import numpy as np
+
+ cards = [
+ emb.Card(1, "podcasts", "Podcast", "One", "", "a", 200, 20, False, False, ["English"]),
+ emb.Card(2, "publications", "Publication", "Two", "", "b", 200, 20, False, False, ["French"]),
+ emb.Card(3, "projects", "Project", "Three", "", "c", 20, 2, False, True, ["English"]),
+ ]
+ matrix = np.asarray([[1.0, 0.0], [0.9, 0.1], [1.0, 0.0]], dtype=np.float32)
+ matrix /= np.linalg.norm(matrix, axis=1, keepdims=True)
+ similar = emb.similar_items(cards, matrix, top_k=2)
+ self.assertEqual({"1", "2"}, set(similar))
+ self.assertEqual(2, similar["1"][0]["id"])
+ self.assertNotIn(3, [row["id"] for neighbours in similar.values() for row in neighbours])
+
+ def test_two_record_projection_does_not_enter_invalid_umap_mode(self):
+ import numpy as np
+
+ coords, clusters, count = emb.project_and_cluster(
+ np.asarray([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32)
+ )
+ self.assertEqual((2, 2), coords.shape)
+ self.assertEqual([0, 1], clusters.tolist())
+ self.assertEqual(2, count)
+
+ def test_projection_smoke_uses_the_pinned_umap_stack(self):
+ import numpy as np
+
+ matrix = np.random.default_rng(42).normal(size=(12, 6)).astype(np.float32)
+ matrix /= np.linalg.norm(matrix, axis=1, keepdims=True)
+ coords, clusters, count = emb.project_and_cluster(matrix)
+ self.assertEqual((12, 2), coords.shape)
+ self.assertTrue(np.isfinite(coords).all())
+ self.assertEqual(12, len(clusters))
+ self.assertGreaterEqual(count, 2)
+
+ def test_release_vectors_are_little_endian_and_checksummed(self):
+ import numpy as np
+
+ old_dir, old_dims = emb.RELEASE_DIR, emb.OUTPUT_DIMS
+ try:
+ with tempfile.TemporaryDirectory() as directory:
+ emb.RELEASE_DIR = Path(directory)
+ emb.OUTPUT_DIMS = 2
+ cards = [emb.Card(7, "projects", "Project", "Seven", "", "h", 200, 20, False, False, [])]
+ manifest = emb.write_release(np.asarray([[0.25, 0.75]], dtype=np.float32), cards, "2026-01-01T00:00:00Z")
+ raw = (Path(directory) / "vectors.f32").read_bytes()
+ self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest["vectorsSha256"])
+ self.assertEqual("float32-le", manifest["dtype"])
+ self.assertEqual(8, len(raw))
+ self.assertEqual(7, json.loads((Path(directory) / "ids.json").read_text())[0]["id"])
+ finally:
+ emb.RELEASE_DIR, emb.OUTPUT_DIMS = old_dir, old_dims
+
+ def test_derived_artifacts_match_the_frontend_schema(self):
+ import numpy as np
+
+ old_out, old_release, old_dims = emb.OUT_DIR, emb.RELEASE_DIR, emb.OUTPUT_DIMS
+ try:
+ with tempfile.TemporaryDirectory() as directory:
+ emb.OUT_DIR = Path(directory)
+ emb.RELEASE_DIR = Path(directory) / "release"
+ emb.OUTPUT_DIMS = 2
+ cards = [
+ emb.Card(1, "podcasts", "Podcast", "Episode", "", "a", 200, 20, False, False, ["English"]),
+ emb.Card(2, "publications", "Publication", "Article", "", "b", 200, 20, False, False, ["French"]),
+ ]
+ report = emb.write_derived(
+ cards,
+ np.asarray([[1.0, 0.0], [0.8, 0.2]], dtype=np.float32),
+ np.asarray([[-1.0, 0.0], [1.0, 0.0]], dtype=np.float32),
+ np.asarray([0, 1], dtype=np.int32),
+ 2,
+ {"1": [{"id": 2, "score": 0.8}], "2": [{"id": 1, "score": 0.8}]},
+ {
+ "podcasts": {"fetched": 1, "public": 1, "nonPublicSkipped": 0, "cards": 1, "lowSignal": 0, "truncated": 0},
+ "publications": {"fetched": 1, "public": 1, "nonPublicSkipped": 0, "cards": 1, "lowSignal": 0, "truncated": 0},
+ },
+ {"embedded": 2, "reused": 0, "staleRemoved": 0},
+ )
+ map_data = json.loads((Path(directory) / "map.json").read_text())
+ similar_data = json.loads((Path(directory) / "similar.json").read_text())
+ self.assertEqual(1, map_data["schemaVersion"])
+ self.assertEqual(
+ {"id", "x", "y", "type", "typeLabel", "cluster", "lowSignal", "title"},
+ set(map_data["items"][0]),
+ )
+ self.assertEqual("Episode", similar_data["catalog"]["1"]["title"])
+ self.assertEqual(2, similar_data["items"]["1"][0]["id"])
+ self.assertEqual(2, report["totals"]["cards"])
+ self.assertEqual(
+ {"cards": 2, "recommendationSources": 2, "lowSignal": 0},
+ emb.validate_artifacts(emb.OUT_DIR, emb.RELEASE_DIR),
+ )
+ finally:
+ emb.OUT_DIR, emb.RELEASE_DIR, emb.OUTPUT_DIMS = old_out, old_release, old_dims
+
+ def test_artifact_validation_rejects_a_tampered_vector_release(self):
+ import numpy as np
+
+ old_out, old_release, old_dims = emb.OUT_DIR, emb.RELEASE_DIR, emb.OUTPUT_DIMS
+ try:
+ with tempfile.TemporaryDirectory() as directory:
+ emb.OUT_DIR = Path(directory)
+ emb.RELEASE_DIR = Path(directory) / "release"
+ emb.OUTPUT_DIMS = 2
+ cards = [
+ emb.Card(1, "projects", "Project", "One", "", "a", 200, 20, False, False, []),
+ emb.Card(2, "sections", "Section", "Two", "", "b", 200, 20, False, False, []),
+ ]
+ emb.write_derived(
+ cards,
+ np.asarray([[1.0, 0.0], [0.8, 0.2]], dtype=np.float32),
+ np.asarray([[-1.0, 0.0], [1.0, 0.0]], dtype=np.float32),
+ np.asarray([0, 1], dtype=np.int32),
+ 2,
+ {"1": [{"id": 2, "score": 0.8}], "2": [{"id": 1, "score": 0.8}]},
+ {},
+ {"embedded": 2, "reused": 0, "staleRemoved": 0},
+ )
+ (emb.RELEASE_DIR / "vectors.f32").write_bytes(b"tampered")
+ with self.assertRaisesRegex(ValueError, "Vector byte length"):
+ emb.validate_artifacts(emb.OUT_DIR, emb.RELEASE_DIR)
+ finally:
+ emb.OUT_DIR, emb.RELEASE_DIR, emb.OUTPUT_DIMS = old_out, old_release, old_dims
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/view/common/block-layout/semantic-map.phtml b/view/common/block-layout/semantic-map.phtml
new file mode 100644
index 00000000..28533cb3
--- /dev/null
+++ b/view/common/block-layout/semantic-map.phtml
@@ -0,0 +1,28 @@
+dashboardAssets(['cdn' => true, 'semanticMap' => true, 'controller' => 'semanticMap']);
+$siteSlug = $this->currentSite()->slug();
+?>
+
+
+
+
+
+
+
= $this->translate('Loading the semantic map…') ?>
+
+
+
diff --git a/view/common/resource-page-block-layout/similar-items.phtml b/view/common/resource-page-block-layout/similar-items.phtml
new file mode 100644
index 00000000..bd0ba484
--- /dev/null
+++ b/view/common/resource-page-block-layout/similar-items.phtml
@@ -0,0 +1,28 @@
+headScript()->appendFile(
+ $this->assetUrl('js/semantic-similar.js', 'DreVisualizations'),
+ 'text/javascript',
+ ['defer' => true]
+);
+$siteSlug = $this->currentSite()->slug();
+?>
+
+
+ = $this->translate('Related by meaning') ?>
+ = $this->translate('Explore records with similar subjects and descriptions, even when their language or resource type differs.') ?>
+
+