feat(favicons): site icons on the tabs and the new tab, kept inside the app

app/src/assets/home.css

@@ -113,6 +113,9 @@ main {
   font-style: normal;
   font-weight: 700;
 }
+/* Site icons are drawn for a light page (GitHub's is black): they sit on a light circle so every one stays visible. */
+.card i.has-icon { background: #f4f5f7; }
+.card i img { width: calc(28 * var(--u)); height: calc(28 * var(--u)); object-fit: contain; border-radius: calc(4 * var(--u)); }
 .card span { max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
 
 footer { position: fixed; bottom: var(--space-4); left: 0; right: 0; display: flex; align-items: center; gap: var(--space-4); justify-content: center; color: var(--faint); font-size: var(--text-xs); flex-wrap: wrap; }

app/src/assets/newtab.html

@@ -48,14 +48,30 @@
   let recent = [];
   let showSites = true;
 
+  // The site's own icon, kept inside the app; the initial in the circle stays until an icon loads (or when there is none).
+  let refreshed = 0;
+  let faviconKey = '';
+  const hostOf = (url) => { try { const u = new URL(url); return u.protocol === 'https:' || u.protocol === 'http:' ? u.hostname : ''; } catch { return ''; } };
+
   function card(name, url) {
     const b = document.createElement('button');
     b.className = 'card';
     b.title = url;
     const badge = document.createElement('i');
     // The initial is decoration: the button is named by the title alone, not "Y YouTube TV".
     badge.setAttribute('aria-hidden', 'true');
-    badge.textContent = name.trim()[0]?.toUpperCase() ?? '?';
+    // The letter is its own text node, so taking it away when the icon arrives leaves the icon in place.
+    const letter = document.createTextNode(name.trim()[0]?.toUpperCase() ?? '?');
+    badge.append(letter);
+    const host = hostOf(url);
+    if (host) {
+      const icon = document.createElement('img');
+      icon.alt = '';
+      icon.addEventListener('load', () => { letter.remove(); badge.classList.add('has-icon'); });
+      icon.addEventListener('error', () => icon.remove());
+      icon.src = `joychromium://favicon/?host=${encodeURIComponent(host)}&k=${faviconKey}&r=${refreshed}`;
+      badge.append(icon);
+    }
     const label = document.createElement('span');
     label.textContent = name;
     b.append(badge, label);
@@ -79,24 +95,41 @@
       r.setProperty('--accent-text', (0.2126 * (v >> 16 & 255) + 0.7152 * (v >> 8 & 255) + 0.0722 * (v & 255)) / 255 > 0.45 ? '#0A0F1A' : '#FFFFFF');
       $('engine').textContent = `${msg.searchEngine.name}`;
       document.body.dataset.bg = msg.newTabBackground || 'Wallpaper';
+      faviconKey = msg.faviconKey || '';
       const b = parseInt(t.background.slice(1), 16);
       document.body.dataset.tone = (0.2126 * (b >> 16 & 255) + 0.7152 * (b >> 8 & 255) + 0.0722 * (b & 255)) / 255 > 0.5 ? 'light' : 'dark';
       showSites = msg.newTabSites !== false;
       pinned = [...msg.shortcuts, ...(msg.favorites || [])];
       render();
     }
+    if (msg.type === 'favicons') {
+      // Icons arrived for sites that had none: draw the tiles again so those load.
+      refreshed += 1;
+      render();
+    }
     if (msg.type === 'history' && Array.isArray(msg.entries)) {
       recent = msg.entries.map((e) => ({ name: e.title || e.url, url: e.url }));
       render();
     }
   });
 
   function render() {
-    const seen = new Set();
-    const sites = [...pinned, ...recent].filter((s) => !seen.has(s.url) && seen.add(s.url)).slice(0, MAX_TILES);
+    // The user's own shortcuts and favorites always show; a recent page is left out when its site is already there, so a
+    // site is one tile whatever page of it was opened last.
+    const siteOf = (url) => hostOf(url).replace(/^www\./, '') || url;
+    const urls = new Set();
+    const own = pinned.filter((s) => !urls.has(s.url) && urls.add(s.url));
+    const taken = new Set(own.map((s) => siteOf(s.url)));
+    const later = recent.filter((s) => !taken.has(siteOf(s.url)) && taken.add(siteOf(s.url)));
+    const sites = [...own, ...later].slice(0, MAX_TILES);
     $('sites').hidden = !showSites || sites.length === 0;
     $('sites').replaceChildren(...sites.map((s) => card(s.name, s.url)));
+    // Sites with no icon kept yet (visited before icons were saved, or only added by hand): ask for them once.
+    const wanted = [...new Set(sites.map((s) => hostOf(s.url)).filter((h) => h && !asked.has(h)))];
+    for (const h of wanted) asked.add(h);
+    if (wanted.length) post('favicon-fetch', { hosts: wanted });
   }
+  const asked = new Set();
   post('ready', {});
   post('history-search', { query: '' });
 </script>

app/src/main/adblock.ts

@@ -2,7 +2,7 @@ import { Request } from "@ghostery/adblocker";
 import { ElectronBlocker } from "@ghostery/adblocker-electron";
 import { promises as fs } from "node:fs";
 import { join } from "node:path";
-import type { Session } from "electron";
+import { ipcMain, type Session } from "electron";
 import type { TrackingLevel } from "../shared/settings";
 import { trackerListsFor } from "../shared/trackerLists";
 
@@ -68,9 +68,20 @@ export class AdBlock {
     });
   }
 
+  /**
+   * The engine registers its cosmetic-filter handlers on two channels of ipcMain, which are global: turning it on in a
+   * second session (a private tab's own) throws "Attempted to register a second handler" and the tab never opens. The
+   * handlers belong to the one engine whatever the session, so the earlier ones are dropped first.
+   */
+  private enableIn(engine: ElectronBlocker, session: Session): void {
+    ipcMain.removeHandler("@ghostery/adblocker/inject-cosmetic-filters");
+    ipcMain.removeHandler("@ghostery/adblocker/is-mutation-observer-enabled");
+    engine.enableBlockingInSession(session);
+  }
+
   enable(session: Session): void {
     if (!this.blocker || this.enabledOn.has(session)) return;
-    this.blocker.enableBlockingInSession(session);
+    this.enableIn(this.blocker, session);
     this.enabledOn.add(session);
   }
 
@@ -100,7 +111,7 @@ export class AdBlock {
   private install(engine: ElectronBlocker): void {
     for (const session of this.enabledOn) this.blocker?.disableBlockingInSession(session);
     this.blocker = engine;
-    for (const session of this.enabledOn) engine.enableBlockingInSession(session);
+    for (const session of this.enabledOn) this.enableIn(engine, session);
   }
 
   private cachePath(level: TrackingLevel): string {

app/src/main/favicons.ts

@@ -0,0 +1,200 @@
+import { randomBytes } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+/** Site icons kept inside the app: the tab strip and the new tab read them from here, never from the site. */
+export const FAVICON_MAX_BYTES = 128 * 1024;
+const FAVICON_TIMEOUT_MS = 8000;
+
+const EXTENSIONS: Record<string, string> = {
+  "image/png": "png", "image/x-icon": "ico", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif", "image/svg+xml": "svg",
+};
+const MIME_OF = Object.fromEntries(Object.entries(EXTENSIONS).map(([mime, ext]) => [ext, mime]));
+
+export interface Favicon {
+  mime: string;
+  bytes: Buffer;
+}
+
+/** The host a favicon belongs to, lowercased, or undefined for anything that is not a plain web host. */
+export function faviconHost(url: string | undefined | null): string | undefined {
+  if (!url) return undefined;
+  try {
+    const parsed = new URL(url);
+    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return undefined;
+    // The host becomes a file name: letters, digits, dots and dashes only (this also leaves out IPv6 literals).
+    return /^[a-z0-9.-]{1,253}$/.test(parsed.hostname) ? parsed.hostname : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** What the bytes really are, whatever the server said: only image types a tab can show are accepted. */
+export function sniffImage(bytes: Uint8Array): string | undefined {
+  const at = (i: number) => bytes[i] ?? -1;
+  if (at(0) === 0x89 && at(1) === 0x50 && at(2) === 0x4e && at(3) === 0x47) return "image/png";
+  if (at(0) === 0 && at(1) === 0 && at(2) === 1 && at(3) === 0) return "image/x-icon";
+  if (at(0) === 0xff && at(1) === 0xd8) return "image/jpeg";
+  if (at(0) === 0x47 && at(1) === 0x49 && at(2) === 0x46) return "image/gif";
+  if (at(0) === 0x52 && at(1) === 0x49 && at(2) === 0x46 && at(8) === 0x57 && at(9) === 0x45 && at(10) === 0x42 && at(11) === 0x50) return "image/webp";
+  if (/<svg[\s>]/i.test(Buffer.from(bytes.subarray(0, 512)).toString("utf8"))) return "image/svg+xml";
+  return undefined;
+}
+
+/** Which of the icons a page announced to take: a web icon over a data one, the first of each kind. */
+export function pickFavicon(candidates: readonly string[]): string | undefined {
+  return candidates.find((c) => c.startsWith("https://")) ?? candidates.find((c) => c.startsWith("http://")) ?? candidates.find((c) => c.startsWith("data:image/"));
+}
+
+async function readBody(response: Response): Promise<Buffer | undefined> {
+  const declared = Number(response.headers.get("content-length"));
+  if (Number.isFinite(declared) && declared > FAVICON_MAX_BYTES) return undefined;
+  if (!response.body) return undefined;
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  const reader = response.body.getReader();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    size += value.length;
+    if (size > FAVICON_MAX_BYTES) {
+      await reader.cancel();
+      return undefined;
+    }
+    chunks.push(value);
+  }
+  return Buffer.concat(chunks);
+}
+
+/** Downloads (or decodes) one favicon through `fetchFn`, the session of the tab that asked. Returns undefined on any doubt. */
+export async function readFavicon(url: string, fetchFn: (input: string, init?: { signal?: AbortSignal }) => Promise<Response>): Promise<Favicon | undefined> {
+  try {
+    let bytes: Buffer | undefined;
+    if (url.startsWith("data:image/")) {
+      const comma = url.indexOf(",");
+      if (comma < 0 || url.length > FAVICON_MAX_BYTES * 2) return undefined;
+      const head = url.slice(0, comma);
+      const payload = url.slice(comma + 1);
+      bytes = head.includes(";base64") ? Buffer.from(payload, "base64") : Buffer.from(decodeURIComponent(payload), "utf8");
+    } else {
+      // A plain http icon is asked for over https: the page itself is upgraded the same way.
+      const target = url.replace(/^http:\/\//i, "https://");
+      if (!target.startsWith("https://")) return undefined;
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FAVICON_TIMEOUT_MS);
+      try {
+        const response = await fetchFn(target, { signal: controller.signal });
+        if (!response.ok) return undefined;
+        bytes = await readBody(response);
+      } finally {
+        clearTimeout(timer);
+      }
+    }
+    if (!bytes || bytes.length === 0 || bytes.length > FAVICON_MAX_BYTES) return undefined;
+    const mime = sniffImage(bytes);
+    return mime ? { mime, bytes } : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+interface Entry {
+  mime: string;
+  bytes?: Buffer;
+  file?: string;
+  version: number;
+}
+
+/**
+ * One icon per host. Normal tabs write theirs to the profile (so the new tab can draw them on the next launch);
+ * private tabs keep theirs in memory only, gone when the app closes. At most `maxFiles` icons stay on disk, the
+ * oldest going first.
+ */
+export class FaviconStore {
+  private readonly disk = new Map<string, Entry>();
+  private readonly memory = new Map<string, Entry>();
+  private counter = 1;
+  /**
+   * A random value made at start-up and put in every address handed to our own documents. A web page asking for
+   * joychromium://favicon looks exactly like the shell asking (no referrer, no origin), so without it any page could
+   * probe which sites were visited by checking which icons exist.
+   */
+  readonly key = randomBytes(16).toString("hex");
+
+  constructor(private readonly dir: string, private readonly log: (message: string) => void = () => {}, private readonly maxFiles = 500) {
+    try {
+      const files = existsSync(dir) ? readdirSync(dir) : [];
+      const known = files
+        .map((file) => ({ file, host: file.slice(0, file.lastIndexOf(".")), ext: file.slice(file.lastIndexOf(".") + 1) }))
+        .filter((f) => MIME_OF[f.ext] && /^[a-z0-9.-]{1,253}$/.test(f.host))
+        .map((f) => ({ ...f, time: statSync(join(dir, f.file)).mtimeMs }))
+        .sort((a, b) => a.time - b.time);
+      for (const f of known) this.disk.set(f.host, { mime: MIME_OF[f.ext]!, file: f.file, version: this.counter++ });
+    } catch (error) {
+      this.log(`Favicons: could not read ${dir}: ${String(error)}`);
+    }
+  }
+
+  has(host: string): boolean {
+    return this.memory.has(host) || this.disk.has(host);
+  }
+
+  /** The address the shell and the internal pages use for a host's icon, or undefined when none is kept. */
+  urlFor(host: string | undefined): string | undefined {
+    const entry = host ? this.memory.get(host) ?? this.disk.get(host) : undefined;
+    return entry && host ? `joychromium://favicon/?host=${encodeURIComponent(host)}&v=${entry.version}&k=${this.key}` : undefined;
+  }
+
+  get(host: string): Favicon | undefined {
+    const entry = this.memory.get(host) ?? this.disk.get(host);
+    if (!entry) return undefined;
+    if (!entry.bytes && entry.file) {
+      try {
+        entry.bytes = readFileSync(join(this.dir, entry.file));
+      } catch (error) {
+        this.log(`Favicons: could not read ${entry.file}: ${String(error)}`);
+        this.disk.delete(host);
+        return undefined;
+      }
+    }
+    return entry.bytes ? { mime: entry.mime, bytes: entry.bytes } : undefined;
+  }
+
+  put(host: string, icon: Favicon, persist: boolean): void {
+    const version = this.counter++;
+    if (!persist) {
+      this.memory.set(host, { mime: icon.mime, bytes: icon.bytes, version });
+      return;
+    }
+    const ext = EXTENSIONS[icon.mime];
+    if (!ext) return;
+    try {
+      mkdirSync(this.dir, { recursive: true });
+      const file = `${host}.${ext}`;
+      const old = this.disk.get(host);
+      writeFileSync(join(this.dir, file), icon.bytes);
+      // The same host may come back in another format: the old file must not outlive it.
+      if (old?.file && old.file !== file) rmSync(join(this.dir, old.file), { force: true });
+      this.disk.delete(host);
+      this.disk.set(host, { mime: icon.mime, bytes: icon.bytes, file, version });
+      while (this.disk.size > this.maxFiles) {
+        const [oldest, entry] = this.disk.entries().next().value as [string, Entry];
+        this.disk.delete(oldest);
+        if (entry.file) rmSync(join(this.dir, entry.file), { force: true });
+      }
+    } catch (error) {
+      this.log(`Favicons: could not save ${host}: ${String(error)}`);
+    }
+  }
+
+  /** Forgets every icon, on disk and in memory: part of clearing the browsing data. */
+  clear(): void {
+    this.memory.clear();
+    this.disk.clear();
+    try {
+      rmSync(this.dir, { recursive: true, force: true });
+    } catch (error) {
+      this.log(`Favicons: could not clear ${this.dir}: ${String(error)}`);
+    }
+  }
+}

app/src/main/main.ts

@@ -3,6 +3,7 @@ import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { AdBlock } from "./adblock";
 import { exportDiagnostics } from "./diagnostics";
+import { FaviconStore } from "./favicons";
 import { History } from "./history";
 import { Log } from "./log";
 import { PolicyService } from "./policyService";
@@ -35,7 +36,7 @@ export interface JoyState {
   adblockLevel: () => string | undefined;
   userAgent: string;
   wouldBlock: (url: string) => boolean;
-  tabs: () => { url: string; title: string; isActive: boolean; isLoading: boolean; isPlayingAudio: boolean }[];
+  tabs: () => { url: string; title: string; isActive: boolean; isLoading: boolean; isPlayingAudio: boolean; favicon?: string; isPrivate: boolean }[];
   command: (command: unknown) => Promise<void>;
   /** Sends a controller event through the same router a gamepad uses (tests only). */
   controller: (input: ControllerEvent) => Promise<void>;
@@ -68,6 +69,7 @@ const distDir = join(__dirname, "..");
 const log = new Log(join(dataDir, "logs"));
 const store = new SettingsStore(dataDir, (m) => log.warn(m));
 const history = new History(join(dataDir, "history.jsonl"), (m) => log.warn(m));
+const favicons = new FaviconStore(join(dataDir, "favicons"), (m) => log.warn(m));
 const policy = new PolicyService(dataDir, (m) => log.warn(m));
 const adblock = new AdBlock(dataDir, (m) => log.info(m));
 const updater = new Updater(log, () => policy.current.blockedAppVersions);
@@ -101,7 +103,7 @@ async function start(): Promise<void> {
   const settings = store.load();
   policy.loadCached();
   crashes.sync(settings.crashReports);
-  installProtocolHandler(distDir);
+  installProtocolHandler(distDir, favicons);
 
   const minimum = effectiveMinimumChromium(policy.current, MINIMUM_CHROMIUM);
   if (!versionAtLeast(process.versions.chrome, minimum)) log.warn(`Chromium ${process.versions.chrome} is below the minimum ${minimum}`);
@@ -143,6 +145,7 @@ async function start(): Promise<void> {
   const shell = new Shell({
     store,
     history,
+    favicons,
     policy,
     adblock,
     log,
@@ -167,7 +170,7 @@ async function start(): Promise<void> {
   shellRef = shell;
   globalThis.joy.exportDiagnostics = exportDiagnosticsReport;
   globalThis.joy.tabs = () =>
-    shell.tabs.tabs.map((t) => ({ url: t.url, title: t.title, isActive: t === shell.tabs.active, isLoading: t.isLoading, isPlayingAudio: t.isPlayingAudio }));
+    shell.tabs.tabs.map((t) => ({ url: t.url, title: t.title, isActive: t === shell.tabs.active, isLoading: t.isLoading, isPlayingAudio: t.isPlayingAudio, favicon: t.favicon, isPrivate: t.isPrivate }));
   globalThis.joy.command = (command) => shell.handle(command as never);
   globalThis.joy.controller = (input) => shell.handleController(input);
   globalThis.joy.pageEval = (script) => shell.activeWebContents?.executeJavaScript(script, true) ?? Promise.resolve(undefined);