Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 108 additions & 0 deletions .github/workflows/indexnow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
name: IndexNow ping

# Notify Bing/Yandex/Naver of changed URLs after a successful main-branch
# deploy. Uses the IndexNow open protocol with a key file hosted at
# https://gemmology.dev/<INDEXNOW_KEY>.txt — the key value is stored as a
# repo secret and the public file is generated as part of the build (see
# `public/<key>.txt` placeholder; replace with the real key on first setup).
#
# Diffs the sitemap between the previous main commit and the current commit
# to keep the submission list under IndexNow's 10k-URL limit and avoid
# spam-pinging the entire 600+ mineral catalogue on cosmetic changes.

on:
workflow_run:
workflows: ["Deploy to Cloudflare Pages"]
types: [completed]
branches: [main]
workflow_dispatch:
inputs:
full_resubmit:
description: 'Resubmit every URL in the sitemap (use sparingly)'
type: boolean
default: false

jobs:
ping:
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 50

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'

- name: Determine changed URLs
id: urls
env:
FULL: ${{ github.event.inputs.full_resubmit || 'false' }}
run: |
set -euo pipefail
mkdir -p .indexnow
if [ "$FULL" = "true" ]; then
echo "Submitting full sitemap"
curl -fsSL https://gemmology.dev/sitemap-0.xml \
| grep -oE 'https://gemmology\.dev/[^<]*' \
| sort -u > .indexnow/urls.txt
else
# Diff content/page paths since the previous commit on main.
git diff --name-only "${{ github.event.before || 'HEAD~1' }}"...HEAD \
| grep -E '^(src/pages|src/content|public/crystals)/' \
| sort -u > .indexnow/changed.txt || true
: > .indexnow/urls.txt
while IFS= read -r path; do
case "$path" in
src/pages/index.astro) echo 'https://gemmology.dev/' >> .indexnow/urls.txt ;;
src/pages/about.astro) echo 'https://gemmology.dev/about/' >> .indexnow/urls.txt ;;
src/pages/learn/index.astro) echo 'https://gemmology.dev/learn/' >> .indexnow/urls.txt ;;
src/pages/gallery/index.astro) echo 'https://gemmology.dev/gallery/' >> .indexnow/urls.txt ;;
src/content/learn/*.yaml)
slug="${path#src/content/learn/}"
slug="${slug%.yaml}"
echo "https://gemmology.dev/learn/${slug}/" >> .indexnow/urls.txt
;;
public/crystals/*.svg)
slug="${path#public/crystals/}"
slug="${slug%.svg}"
echo "https://gemmology.dev/minerals/${slug}/" >> .indexnow/urls.txt
;;
esac
done < .indexnow/changed.txt
sort -u -o .indexnow/urls.txt .indexnow/urls.txt
fi
count=$(wc -l < .indexnow/urls.txt | tr -d ' ')
echo "count=$count" >> "$GITHUB_OUTPUT"
echo "Found $count URLs to submit"

- name: Submit to IndexNow
if: ${{ steps.urls.outputs.count != '0' }}
env:
INDEXNOW_KEY: ${{ secrets.INDEXNOW_KEY }}
run: |
if [ -z "${INDEXNOW_KEY:-}" ]; then
echo "INDEXNOW_KEY secret not configured — skipping submission."
exit 0
fi
mapfile -t URLS < .indexnow/urls.txt
# Build JSON array
payload=$(jq -n \
--arg host "gemmology.dev" \
--arg key "$INDEXNOW_KEY" \
--arg keyLocation "https://gemmology.dev/${INDEXNOW_KEY}.txt" \
--argjson urls "$(printf '%s\n' "${URLS[@]}" | jq -R . | jq -s .)" \
'{host:$host, key:$key, keyLocation:$keyLocation, urlList:$urls}')
for endpoint in \
https://api.indexnow.org/IndexNow \
https://www.bing.com/indexnow \
https://yandex.com/indexnow \
https://searchadvisor.naver.com/indexnow ; do
echo "POST $endpoint"
curl -fsS -X POST "$endpoint" \
-H 'Content-Type: application/json; charset=utf-8' \
--data "$payload" || echo " (endpoint unreachable, continuing)"
done
26 changes: 25 additions & 1 deletion astro.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,31 @@ export default defineConfig({
}),
react(),
tailwind(),
sitemap(),
sitemap({
filter: (page) => !page.includes('/admin'),
changefreq: 'weekly',
priority: 0.7,
lastmod: new Date(),
serialize(item) {
// Boost priority/lastmod for hub pages and canonical references
if (item.url === 'https://gemmology.dev/' ||
item.url === 'https://gemmology.dev/gallery/' ||
item.url === 'https://gemmology.dev/learn/') {
item.priority = 1.0;
item.changefreq = 'weekly';
} else if (/\/minerals\/[^/]+\/$/.test(item.url)) {
item.priority = 0.8;
item.changefreq = 'monthly';
} else if (/\/learn\//.test(item.url)) {
item.priority = 0.7;
item.changefreq = 'monthly';
} else if (/\/docs\//.test(item.url)) {
item.priority = 0.5;
item.changefreq = 'monthly';
}
return item;
},
}),
],
vite: {
optimizeDeps: {
Expand Down
20 changes: 20 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@
"preview": "astro preview",
"copy-db": "node scripts/copy-database.js",
"prerender": "node scripts/prerender-crystals.js",
"prebuild": "npm run sync && npm run copy-db",
"check-svgs": "node scripts/audit-crystal-svgs.mjs",
"prebuild": "npm run sync && npm run copy-db && npm run check-svgs",
"generate-og": "npm run build && node scripts/generate-og-image.js"
},
"dependencies": {
"@astrojs/node": "^9.5.2",
"@astrojs/react": "^4.1.0",
"@astrojs/sitemap": "^3.7.0",
"@astrojs/tailwind": "^5.1.0",
"@fontsource-variable/inter": "^5.2.8",
"@fontsource-variable/jetbrains-mono": "^5.2.8",
"@gemmology/mineral-data": "^2.2.1",
"@monaco-editor/react": "^4.6.0",
"@react-three/drei": "^9.105.0",
Expand Down
1 change: 1 addition & 0 deletions public/48f06a3b34104910a4b95d23ad8cb13f.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
48f06a3b34104910a4b95d23ad8cb13f
27 changes: 27 additions & 0 deletions public/_headers
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
X-Content-Type-Options: nosniff
X-Frame-Options: DENY
Referrer-Policy: strict-origin-when-cross-origin
Permissions-Policy: camera=(), microphone=(), geolocation=(), interest-cohort=()
Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; connect-src 'self'; worker-src 'self' blob:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'
Strict-Transport-Security: max-age=31536000; includeSubDomains; preload

# Long-cache versioned static assets (Astro hashed bundle)
/_astro/*
Cache-Control: public, max-age=31536000, immutable

# Crystal SVGs are stable per mineral id
/crystals/*
Cache-Control: public, max-age=604800, must-revalidate

# Static GLB models
/models/*
Cache-Control: public, max-age=604800, must-revalidate

# Database is regenerated on each build — short cache, must-revalidate
/minerals.db
Cache-Control: public, max-age=3600, must-revalidate

# sql.js wasm
/sql.js/*
Cache-Control: public, max-age=2592000, must-revalidate
22 changes: 22 additions & 0 deletions public/_redirects
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Legacy flat learn slugs → topic/slug structure
# Pre-2026 site shipped /learn/<slug>; current structure is /learn/<topic>/<slug>.
/learn/crystal-systems /learn/fundamentals/crystal-systems 301
/learn/crystallography-advanced /learn/fundamentals/crystallography-advanced 301
/learn/physical-properties /learn/fundamentals/physical-properties 301
/learn/optical-properties /learn/fundamentals/optical-properties 301
/learn/optical-properties-advanced /learn/fundamentals/optical-properties-advanced 301
/learn/chemical-properties /learn/fundamentals/chemical-properties 301
/learn/colour-theory /learn/fundamentals/colour-theory 301
/learn/twin-laws /learn/fundamentals/twin-laws 301
/learn/optic-sign-determination /learn/fundamentals/optic-sign-determination 301
/learn/inclusions /learn/identification/inclusions 301
/learn/treatments /learn/identification/treatments 301
/learn/synthetics /learn/identification/synthetics 301

# Old gallery deep links
/minerals/ /gallery/ 301
/gem/* /minerals/:splat 301

# Old docs
/cdl /docs/cdl 301
/api /docs/api 301
Loading
Loading