Merge pull request #30 from JavaScriptSolidServer/issue-29-relay-sweep

scripts: relay-directory data sweep (bad/dup/host-spam/renorm/stale)

Author: melvincarvalho

package.json

@@ -12,7 +12,8 @@
     "index": "node -e \"import('./src/indexer.js').then(m => m.runIndexer())\"",
     "serve": "node -e \"import('./src/server.js').then(m => m.startServer())\"",
     "probe": "node probe.js",
-    "clean:relays": "node scripts/clean-relays.js"
+    "clean:relays": "node scripts/clean-relays.js",
+    "sweep:relays": "node scripts/sweep-relays.js"
   },
   "dependencies": {
     "@noble/curves": "^2.2.0",

scripts/sweep-relays.js

@@ -0,0 +1,34 @@
+// Relay-directory data sweep: drops bad URLs, de-dups, trims per-host
+// path-variant flooding, normalizes survivors, and (optionally) prunes stale.
+// Pure logic lives in planRelaySweep (src/hoses/relays.js); this is just I/O.
+//
+//   node scripts/sweep-relays.js                 # dry run (report only)
+//   node scripts/sweep-relays.js --apply         # execute
+//   node scripts/sweep-relays.js --max-per-host=3 --stale=7 --apply
+import { connect, close } from '../src/db.js';
+import { planRelaySweep } from '../src/hoses/relays.js';
+import 'dotenv/config';
+
+const RELAY_DIRECTORY = process.env.MONGO_RELAY_DIRECTORY_COLLECTION || 'relays';
+const APPLY = process.argv.includes('--apply');
+const numArg = (name, dflt) => { const a = process.argv.find((x) => x.startsWith(`--${name}=`)); return a ? Number(a.split('=')[1]) : dflt; };
+const maxPerHost = numArg('max-per-host', 3);
+const staleDays = numArg('stale', 0);
+
+const db = await connect();
+const col = db.collection(RELAY_DIRECTORY);
+const docs = await col.find({}, { projection: { relay: 1, checksTotal: 1, lastChecked: 1 } }).toArray();
+
+const plan = planRelaySweep(docs, { maxPerHost, staleDays, now: Date.now() });
+const delIds = [...new Set([...plan.bad, ...plan.dups, ...plan.hostSpam, ...(staleDays ? plan.stale : [])].map((d) => d._id))];
+
+console.log(`[sweep] ${docs.length} relays | bad ${plan.bad.length} | dup ${plan.dups.length} | host-spam ${plan.hostSpam.length} | renorm ${plan.renames.length}`
+  + (staleDays ? ` | stale>${staleDays}d ${plan.stale.length}` : '') + ` | keep ${plan.kept}`);
+console.log(`[sweep] maxPerHost=${maxPerHost}  ->  would delete ${delIds.length}, normalize ${plan.renames.length}  (${APPLY ? 'APPLYING' : 'dry run — pass --apply'})`);
+
+if (APPLY) {
+  for (let i = 0; i < delIds.length; i += 1000) await col.deleteMany({ _id: { $in: delIds.slice(i, i + 1000) } });
+  for (const r of plan.renames) await col.updateOne({ _id: r.doc._id }, { $set: { relay: r.canonical } }).catch(() => {});
+  console.log(`[sweep] deleted ${delIds.length}, normalized ${plan.renames.length}; ${await col.estimatedDocumentCount()} remain`);
+}
+await close();

src/hoses/relays.js

@@ -54,6 +54,65 @@ export function safeRelayUrl(url) {
 // Back-compat alias (older callers/tests): same screening as safeRelayUrl.
 export const canonicalizeRelayUrl = safeRelayUrl;
 
+const hostOf = (u) => { try { return new URL(u).host; } catch { return null; } };
+const isOrigin = (u) => { try { const x = new URL(u); return x.pathname === '/' ? 1 : 0; } catch { return 0; } };
+const tms = (x) => (x ? new Date(x).getTime() || 0 : 0);
+
+/**
+ * Plan a relay-directory cleanup (pure — pass docs + options, no I/O). Returns
+ * docs to delete by category plus URL normalizations. Re-runnable / idempotent.
+ *
+ * - bad:     fails safeRelayUrl (malformed / SSRF / non-ws)
+ * - dups:    multiple docs share a canonical URL — keep the richest
+ * - hostSpam: a host has > maxPerHost distinct URLs (path-variant flooding,
+ *             e.g. a bomb) — keep the origin + richest few, drop the rest
+ * - renames: a surviving doc's stored `relay` ≠ its canonical form
+ * - stale:   (if staleDays) a survivor not checked within staleDays
+ */
+export function planRelaySweep(docs, { maxPerHost = 3, staleDays = 0, now = 0 } = {}) {
+  const rich = (a, b) => (b.checksTotal || 0) - (a.checksTotal || 0) || tms(b.lastChecked) - tms(a.lastChecked);
+  // 1. screen + canonicalize, group by canonical URL
+  const byCanon = new Map();
+  const bad = [];
+  for (const d of docs) {
+    const c = d.relay && safeRelayUrl(d.relay);
+    if (!c) { bad.push(d); continue; }
+    (byCanon.get(c) || byCanon.set(c, []).get(c)).push(d);
+  }
+  // 2. dedup: one survivor per canonical (richest history wins)
+  const dups = [];
+  let survivors = [];
+  for (const [canon, ds] of byCanon) {
+    ds.sort(rich);
+    ds[0]._canon = canon;
+    survivors.push(ds[0]);
+    dups.push(...ds.slice(1));
+  }
+  // 3. host cap: trim path-variant flooding per host
+  const hostSpam = [];
+  if (maxPerHost > 0) {
+    const byHost = new Map();
+    for (const s of survivors) { const h = hostOf(s._canon); (byHost.get(h) || byHost.set(h, []).get(h)).push(s); }
+    const kept = [];
+    for (const [, ss] of byHost) {
+      if (ss.length <= maxPerHost) { kept.push(...ss); continue; }
+      ss.sort((a, b) => isOrigin(b._canon) - isOrigin(a._canon) || rich(a, b)); // origin first, then richest
+      kept.push(...ss.slice(0, maxPerHost));
+      hostSpam.push(...ss.slice(maxPerHost));
+    }
+    survivors = kept;
+  }
+  // 4. normalizations + stale among survivors
+  const renames = [];
+  const stale = [];
+  const cutoff = staleDays && now ? now - staleDays * 864e5 : null;
+  for (const s of survivors) {
+    if (s.relay !== s._canon) renames.push({ doc: s, canonical: s._canon });
+    if (cutoff && tms(s.lastChecked) < cutoff) stale.push(s);
+  }
+  return { bad, dups, hostSpam, renames, stale, kept: survivors.length };
+}
+
 export async function ensureRelayDirectoryIndex(db) {
   await db.collection(RELAY_DIRECTORY).createIndex({ relay: 1 })
     .catch((e) => { if (e?.code !== 85 && e?.code !== 86) throw e; });

test/relay-sweep.test.js

@@ -0,0 +1,69 @@
+// planRelaySweep: categorization for the relay-directory data sweep.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { planRelaySweep } from '../src/hoses/relays.js';
+
+const ids = (arr) => arr.map((d) => d._id).sort();
+
+test('bad URLs are dropped', () => {
+  const docs = [
+    { _id: 1, relay: 'wss://relay.damus.io/' },
+    { _id: 2, relay: 'ws://127.0.0.1/' },       // SSRF
+    { _id: 3, relay: 'garbage' },                // malformed
+    { _id: 4, relay: 'wss://nostr.wine/inbox' },
+  ];
+  const p = planRelaySweep(docs, {});
+  assert.deepEqual(ids(p.bad), [2, 3]);
+});
+
+test('dedup keeps the richest of a canonical group', () => {
+  const docs = [
+    { _id: 1, relay: 'wss://nos.lol', checksTotal: 2 },     // canonicalizes to wss://nos.lol/
+    { _id: 2, relay: 'wss://nos.lol/', checksTotal: 9 },    // richest -> keep
+    { _id: 3, relay: 'wss://nos.lol/', checksTotal: 0 },
+  ];
+  const p = planRelaySweep(docs, {});
+  assert.deepEqual(ids(p.dups), [1, 3]);   // two dropped
+  assert.equal(p.kept, 1);
+});
+
+test('host-spam: trims path-variant flooding, keeps origin + richest few', () => {
+  const docs = [
+    { _id: 0, relay: 'wss://nostr.wine/', checksTotal: 1 },        // origin — always kept
+    { _id: 1, relay: 'wss://nostr.wine/echo-zulu', checksTotal: 5 },
+    { _id: 2, relay: 'wss://nostr.wine/flint-cipher', checksTotal: 4 },
+    { _id: 3, relay: 'wss://nostr.wine/raven-warden', checksTotal: 3 },
+    { _id: 4, relay: 'wss://nostr.wine/juliet-jade', checksTotal: 2 },
+  ];
+  const p = planRelaySweep(docs, { maxPerHost: 3 });
+  assert.equal(p.kept, 3);
+  // origin (0) kept; then richest paths (1,2); spam = the rest (3,4)
+  assert.deepEqual(ids(p.hostSpam), [3, 4]);
+});
+
+test('a host under the cap is untouched', () => {
+  const docs = [
+    { _id: 1, relay: 'wss://relay.example.com/inbox' },
+    { _id: 2, relay: 'wss://relay.example.com/outbox' },
+  ];
+  const p = planRelaySweep(docs, { maxPerHost: 3 });
+  assert.deepEqual(p.hostSpam, []);
+  assert.equal(p.kept, 2);
+});
+
+test('renorm flags survivors whose stored URL is not canonical', () => {
+  const docs = [{ _id: 1, relay: 'wss://nos.lol' }]; // -> wss://nos.lol/
+  const p = planRelaySweep(docs, {});
+  assert.equal(p.renames.length, 1);
+  assert.equal(p.renames[0].canonical, 'wss://nos.lol/');
+});
+
+test('stale prune by age (only when staleDays set)', () => {
+  const now = 1_000_000_000_000;
+  const docs = [
+    { _id: 1, relay: 'wss://a.example.com/', lastChecked: new Date(now - 2 * 864e5).toISOString() }, // 2d old
+    { _id: 2, relay: 'wss://b.example.com/', lastChecked: new Date(now - 10 * 864e5).toISOString() }, // 10d old
+  ];
+  assert.deepEqual(ids(planRelaySweep(docs, { staleDays: 7, now }).stale), [2]);
+  assert.deepEqual(planRelaySweep(docs, { now }).stale, []); // no staleDays -> none
+});