Merge pull request #32 from JavaScriptSolidServer/issue-31-safeurl-concat

Relay cleanup: reject concat junk + sweep --real-only prune

Author: melvincarvalho

scripts/sweep-relays.js

@@ -5,24 +5,27 @@
 //   node scripts/sweep-relays.js                 # dry run (report only)
 //   node scripts/sweep-relays.js --apply         # execute
 //   node scripts/sweep-relays.js --max-per-host=3 --stale=7 --apply
+//   node scripts/sweep-relays.js --real-only --apply   # keep only online-ever bare origins
 import { connect, close } from '../src/db.js';
 import { planRelaySweep } from '../src/hoses/relays.js';
 import 'dotenv/config';
 
 const RELAY_DIRECTORY = process.env.MONGO_RELAY_DIRECTORY_COLLECTION || 'relays';
 const APPLY = process.argv.includes('--apply');
+const realOnly = process.argv.includes('--real-only');
 const numArg = (name, dflt) => { const a = process.argv.find((x) => x.startsWith(`--${name}=`)); return a ? Number(a.split('=')[1]) : dflt; };
 const maxPerHost = numArg('max-per-host', 3);
 const staleDays = numArg('stale', 0);
 
 const db = await connect();
 const col = db.collection(RELAY_DIRECTORY);
-const docs = await col.find({}, { projection: { relay: 1, checksTotal: 1, lastChecked: 1 } }).toArray();
+const docs = await col.find({}, { projection: { relay: 1, checksTotal: 1, checksOnline: 1, lastChecked: 1 } }).toArray();
 
-const plan = planRelaySweep(docs, { maxPerHost, staleDays, now: Date.now() });
-const delIds = [...new Set([...plan.bad, ...plan.dups, ...plan.hostSpam, ...(staleDays ? plan.stale : [])].map((d) => d._id))];
+const plan = planRelaySweep(docs, { maxPerHost, staleDays, realOnly, now: Date.now() });
+const delIds = [...new Set([...plan.bad, ...plan.dups, ...plan.hostSpam, ...plan.notReal, ...(staleDays ? plan.stale : [])].map((d) => d._id))];
 
-console.log(`[sweep] ${docs.length} relays | bad ${plan.bad.length} | dup ${plan.dups.length} | host-spam ${plan.hostSpam.length} | renorm ${plan.renames.length}`
+console.log(`[sweep] ${docs.length} relays | bad ${plan.bad.length} | dup ${plan.dups.length} | host-spam ${plan.hostSpam.length}`
+  + (realOnly ? ` | not-real ${plan.notReal.length}` : '') + ` | renorm ${plan.renames.length}`
   + (staleDays ? ` | stale>${staleDays}d ${plan.stale.length}` : '') + ` | keep ${plan.kept}`);
 console.log(`[sweep] maxPerHost=${maxPerHost}  ->  would delete ${delIds.length}, normalize ${plan.renames.length}  (${APPLY ? 'APPLYING' : 'dry run — pass --apply'})`);
 

src/hoses/relays.js

@@ -43,8 +43,13 @@ function isBlockedHost(host) {
  * (trailing slash on bare origins), or null if it must not be stored/dialed.
  */
 export function safeRelayUrl(url) {
+  const raw = String(url);
+  // Real relay URLs are short and single-scheme. Reject length blowups,
+  // whitespace, and a second "://" (embedded scheme) — the signature of
+  // concatenated multi-relay junk packed into one tag by broken clients.
+  if (raw.length > 120 || /\s/.test(raw) || raw.indexOf('://') !== raw.lastIndexOf('://')) return null;
   let u;
-  try { u = new URL(String(url)); } catch { return null; }
+  try { u = new URL(raw); } catch { return null; }
   if (u.protocol !== 'wss:' && u.protocol !== 'ws:') return null;
   if (isBlockedHost(u.hostname)) return null;
   if (u.pathname === '' || u.pathname === '/') return `${u.protocol}//${u.host}/`;
@@ -68,8 +73,10 @@ const tms = (x) => (x ? new Date(x).getTime() || 0 : 0);
  *             e.g. a bomb) — keep the origin + richest few, drop the rest
  * - renames: a surviving doc's stored `relay` ≠ its canonical form
  * - stale:   (if staleDays) a survivor not checked within staleDays
+ * - notReal: (if realOnly) survivors that aren't a bare origin that has been
+ *            online at least once — i.e. the unverified/dead/path entries
  */
-export function planRelaySweep(docs, { maxPerHost = 3, staleDays = 0, now = 0 } = {}) {
+export function planRelaySweep(docs, { maxPerHost = 3, staleDays = 0, now = 0, realOnly = false } = {}) {
   const rich = (a, b) => (b.checksTotal || 0) - (a.checksTotal || 0) || tms(b.lastChecked) - tms(a.lastChecked);
   // 1. screen + canonicalize, group by canonical URL
   const byCanon = new Map();
@@ -102,15 +109,26 @@ export function planRelaySweep(docs, { maxPerHost = 3, staleDays = 0, now = 0 }
     }
     survivors = kept;
   }
-  // 4. normalizations + stale among survivors
+  // 4. realOnly: keep only bare-origin relays online at least once (the
+  //    verified/reachable set); everything else is unverified/dead/path junk.
+  const notReal = [];
+  if (realOnly) {
+    const real = [];
+    for (const s of survivors) {
+      if (isOrigin(s._canon) && (s.checksOnline || 0) >= 1) real.push(s);
+      else notReal.push(s);
+    }
+    survivors = real;
+  }
+  // 5. normalizations + stale among final survivors
   const renames = [];
   const stale = [];
   const cutoff = staleDays && now ? now - staleDays * 864e5 : null;
   for (const s of survivors) {
     if (s.relay !== s._canon) renames.push({ doc: s, canonical: s._canon });
     if (cutoff && tms(s.lastChecked) < cutoff) stale.push(s);
   }
-  return { bad, dups, hostSpam, renames, stale, kept: survivors.length };
+  return { bad, dups, hostSpam, notReal, renames, stale, kept: survivors.length };
 }
 
 export async function ensureRelayDirectoryIndex(db) {

test/relay-sweep.test.js

@@ -58,6 +58,18 @@ test('renorm flags survivors whose stored URL is not canonical', () => {
   assert.equal(p.renames[0].canonical, 'wss://nos.lol/');
 });
 
+test('realOnly: keep only bare origins online at least once', () => {
+  const docs = [
+    { _id: 1, relay: 'wss://good.example.com/', checksOnline: 3 },     // bare + online -> keep
+    { _id: 2, relay: 'wss://dead.example.com/', checksOnline: 0 },     // bare but never online -> notReal
+    { _id: 3, relay: 'wss://path.example.com/echo-zulu', checksOnline: 5 }, // online but has path -> notReal
+    { _id: 4, relay: 'wss://fresh.example.com/' },                     // no checksOnline -> notReal
+  ];
+  const p = planRelaySweep(docs, { realOnly: true });
+  assert.equal(p.kept, 1);
+  assert.deepEqual(ids(p.notReal), [2, 3, 4]);
+});
+
 test('stale prune by age (only when staleDays set)', () => {
   const now = 1_000_000_000_000;
   const docs = [