Phase 2: the model (weighted vote, confidence, accuracy check)

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ concurrency:
 
 jobs:
   check:
-    name: Typecheck, lint, frontend build
+    name: Typecheck, lint, tests, frontend build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
@@ -37,6 +37,9 @@ jobs:
       - name: Lint
         run: npm run lint
 
+      - name: Test
+        run: npm test
+
       - name: Build frontend
         run: npm run build -w app
 

app/package.json

@@ -8,7 +8,8 @@
     "build": "tsc && vite build",
     "typecheck": "tsc --noEmit",
     "preview": "vite preview",
-    "tauri": "tauri"
+    "tauri": "tauri",
+    "test": "vitest run"
   },
   "dependencies": {
     "@base-ui/react": "^1.8.0",
@@ -35,6 +36,7 @@
     "shadcn": "^4.21.1",
     "tailwindcss": "^4.3.3",
     "typescript": "~7.0.2",
-    "vite": "^8.0.16"
+    "vite": "^8.0.16",
+    "vitest": "^5.0.4"
   }
 }

app/src-tauri/src/data.rs

@@ -6,8 +6,10 @@ use serde::Serialize;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 use strata_db::db::DB_FILE_NAME;
-use strata_db::models::{Borewell, PipeSegment, SearchFilters, StrataLayer, WaterReading};
-use strata_db::repo::{borewells, layers};
+use strata_db::models::{
+    Borewell, Material, PipeSegment, SearchFilters, StrataLayer, WaterReading,
+};
+use strata_db::repo::{borewells, layers, materials};
 use strata_db::schema;
 
 /// Name of the folder, inside the roaming app-data folder, that every Strata app shares.
@@ -23,6 +25,15 @@ pub struct SiteRecord {
     pub water_readings: Vec<WaterReading>,
 }
 
+/// Everything read in one go, all as it was at the same moment.
+#[derive(Debug, Clone, Serialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct Records {
+    pub sites: Vec<SiteRecord>,
+    /// Every soil type, built-in and added by hand; layers refer to them by id.
+    pub materials: Vec<Material>,
+}
+
 /// Why the records could not be read. The screens choose the wording from `kind`.
 #[derive(Debug, Clone, Serialize, PartialEq)]
 #[serde(tag = "kind", rename_all = "camelCase")]
@@ -55,11 +66,11 @@ pub fn data_dir(roaming_app_data: &Path) -> PathBuf {
     }
 }
 
-/// Every borewell that is not in the Recycle bin, read from `<data_dir>/strata.db` as it is at
+/// Every borewell that is not in the Recycle bin, and the soil types, read from `<data_dir>/strata.db` as it is at
 /// this moment. The file is opened read-only: the records are never upgraded or changed, and the
 /// folder and the file are never created. SQLite may add its two bookkeeping files (`-wal` and
 /// `-shm`) beside an existing database, exactly as StrataField itself does.
-pub fn load(data_dir: &Path) -> Result<Vec<SiteRecord>, LoadError> {
+pub fn load(data_dir: &Path) -> Result<Records, LoadError> {
     let path = data_dir.join(DB_FILE_NAME);
     if !path.exists() {
         return Err(LoadError::NoDatabase);
@@ -87,7 +98,10 @@ pub fn load(data_dir: &Path) -> Result<Vec<SiteRecord>, LoadError> {
             strata: item.strata,
         });
     }
-    Ok(sites)
+    Ok(Records {
+        sites,
+        materials: materials::list(&tx)?,
+    })
 }
 
 #[cfg(test)]
@@ -202,7 +216,13 @@ mod tests {
         assert!(!tmp.path().join("strata.db-shm").exists());
         let before = std::fs::read(&db_file).unwrap();
 
-        let got = load(tmp.path()).unwrap();
+        let records = load(tmp.path()).unwrap();
+        // The soil types come with the records: hand-added ones are only known to the database.
+        assert!(records
+            .materials
+            .iter()
+            .any(|m| m.id == "clay" && m.lithology_class.as_deref() == Some("CLAY")));
+        let got = records.sites;
         assert_eq!(got.len(), 2);
         let full = site(&got, "BW-1");
         assert_eq!(full.strata.len(), 1);
@@ -226,7 +246,7 @@ mod tests {
     fn loads_while_another_app_has_it_open() {
         let tmp = tempfile::tempdir().unwrap();
         let _open = seeded(tmp.path());
-        assert_eq!(load(tmp.path()).unwrap().len(), 2);
+        assert_eq!(load(tmp.path()).unwrap().sites.len(), 2);
     }
 
     #[test]
@@ -243,7 +263,7 @@ mod tests {
         })
         .unwrap();
         drop(db);
-        let got = load(tmp.path()).unwrap();
+        let got = load(tmp.path()).unwrap().sites;
         assert_eq!(got.len(), 1);
         assert_eq!(got[0].borewell.borewell_id, "BW-2");
     }

app/src-tauri/src/lib.rs

@@ -2,9 +2,9 @@ mod data;
 
 use tauri::Manager;
 
-/// Every borewell in StrataField's records, read afresh each time the screens ask.
+/// StrataField's records, read afresh each time the screens ask.
 #[tauri::command]
-fn load_data(app: tauri::AppHandle) -> Result<Vec<data::SiteRecord>, data::LoadError> {
+fn load_data(app: tauri::AppHandle) -> Result<data::Records, data::LoadError> {
     let roaming = app.path().data_dir()?;
     data::load(&data::data_dir(&roaming))
 }

app/src/App.tsx

@@ -2,10 +2,10 @@ import { useCallback, useEffect, useState } from "react";
 import { RefreshCw } from "lucide-react";
 import { Button } from "@/components/ui/button";
 import { SiteMap, located } from "@/components/SiteMap";
-import { loadData, type LoadError, type SiteRecord } from "@/lib/api";
+import { loadData, type LoadError, type Records } from "@/lib/api";
 import { text } from "@/text";
 
-type State = { sites: SiteRecord[] } | { problem: LoadError } | null;
+type State = { records: Records } | { problem: LoadError } | null;
 
 export default function App() {
   const [state, setState] = useState<State>(null);
@@ -14,7 +14,7 @@ export default function App() {
   // Reading again keeps what is on screen until the new records arrive, so the map does not blink.
   const load = useCallback(() => {
     loadData().then(
-      (sites) => setState({ sites }),
+      (records) => setState({ records }),
       (e) => setState({ problem: isLoadError(e) ? e : { kind: "other", message: String(e) } }),
     );
   }, []);
@@ -43,7 +43,7 @@ function Body({ state, selectedId, onSelect }: { state: State; selectedId: strin
     const p = state.problem;
     return <Message detail={p.kind === "other" ? p.message : undefined}>{text.problems[p.kind]}</Message>;
   }
-  const { sites } = state;
+  const { sites } = state.records;
   if (sites.length === 0) return <Message>{text.empty}</Message>;
 
   const shown = located(sites).length;

app/src/lib/api.ts

@@ -1,5 +1,5 @@
 import { invoke } from "@tauri-apps/api/core";
-import type { Borewell, PipeSegment, StrataLayer, WaterReading } from "@strata/core";
+import type { Borewell, Material, PipeSegment, StrataLayer, WaterReading } from "@strata/core";
 
 /** One borewell with everything the analysis uses, as read from StrataField's records. */
 export interface SiteRecord {
@@ -9,8 +9,15 @@ export interface SiteRecord {
   waterReadings: WaterReading[];
 }
 
+/** Everything read in one go, all as it was at the same moment. */
+export interface Records {
+  sites: SiteRecord[];
+  /** Every soil type, built-in and added by hand; layers refer to them by id. */
+  materials: Material[];
+}
+
 /** Why the records could not be read (see `LoadError` in src-tauri/src/data.rs). */
 export type LoadError = { kind: "noDatabase" | "newer" | "older" } | { kind: "other"; message: string };
 
-/** Every borewell that is not in the Recycle bin. Rejects with a `LoadError`. */
-export const loadData = () => invoke<SiteRecord[]>("load_data");
+/** Every borewell that is not in the Recycle bin, and the soil types. Rejects with a `LoadError`. */
+export const loadData = () => invoke<Records>("load_data");

app/src/model/check.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it } from "vitest";
+import { DEFAULT_MATERIALS } from "@strata/core";
+import { checkModel, share } from "./check";
+import { prepare, type ModelSite } from "./model";
+import { east, flatGrid, north, site } from "./testGround";
+
+const flat = flatGrid(120);
+const CLAY_OVER_SAND: [number, number, string][] = [[0, 50, "clay"], [50, 100, "medium_sand"]];
+const SAND_OVER_CLAY: [number, number, string][] = [[0, 50, "medium_sand"], [50, 100, "clay"]];
+
+/** `side` × `side` borewells `km` apart, all with the same layers. */
+const town = (side: number, km: number, layers = CLAY_OVER_SAND): ModelSite[] =>
+  Array.from({ length: side * side }, (_, i) => site(`t${i}`, north(Math.floor(i / side) * km), east((i % side) * km), layers));
+const check = (sites: ModelSite[]) => checkModel(prepare(sites, DEFAULT_MATERIALS, flat), flat);
+
+describe("checkModel", () => {
+  it("scores a uniform town as always right", () => {
+    const r = check(town(4, 0.5));
+    expect(r.canJudge).toBe(true);
+    expect(r.compared).toBe(16);
+    expect(r.overall).toEqual({ steps: 320, right: 320 });
+    expect(r.likely).toEqual(r.overall);
+    expect(r.byDistance.under1).toEqual(r.overall);
+    expect(r.oddOnesOut).toEqual([]);
+  });
+
+  it("finds a borewell that does not fit its surroundings", () => {
+    const sites = town(4, 0.5);
+    sites[5] = site("upside-down", sites[5].borewell.latitude, sites[5].borewell.longitude, SAND_OVER_CLAY);
+    const r = check(sites);
+    expect(r.oddOnesOut).toHaveLength(1);
+    expect(r.oddOnesOut[0]).toMatchObject({ id: "upside-down", borewellId: "UPSIDE-DOWN", steps: 20, right: 0, typical: 1 });
+    expect(share(r.overall)).toBe(300 / 320);
+  });
+
+  it("lists the worst first", () => {
+    const sites = town(5, 0.5);
+    sites[6] = site("all-wrong", sites[6].borewell.latitude, sites[6].borewell.longitude, SAND_OVER_CLAY);
+    // Right for the top 30 ft only: 6 steps of 20.
+    sites[18] = site("mostly-wrong", sites[18].borewell.latitude, sites[18].borewell.longitude, [[0, 30, "clay"], [30, 50, "medium_sand"], [50, 100, "clay"]]);
+    expect(check(sites).oddOnesOut.map((o) => o.id)).toEqual(["all-wrong", "mostly-wrong"]);
+  });
+
+  it("hides everything at the spot, so borewells side by side cannot vouch for each other", () => {
+    const sites = town(4, 0.5);
+    const b = sites[5].borewell;
+    sites[5] = site("twin-a", b.latitude, b.longitude, SAND_OVER_CLAY);
+    sites.push(site("twin-b", b.latitude! + 0.03 / 110.57, b.longitude, SAND_OVER_CLAY));
+    expect(check(sites).oddOnesOut.map((o) => o.id).sort()).toEqual(["twin-a", "twin-b"]);
+  });
+
+  it("does not blame a borewell for what lies far below its neighbour", () => {
+    // An honest borewell 4 km out of town in different ground, with a shallow one beside it.
+    const far = east(1.5 + 4);
+    const r = check([...town(4, 0.5), site("edge-deep", north(0), far, [[0, 30, "clay"], [30, 100, "kankar"]]), site("edge-shallow", north(0.03), far, [[0, 30, "clay"]])]);
+    expect(r.oddOnesOut).toEqual([]);
+    expect(r.byDistance.under1).toEqual({ steps: 320, right: 320 });
+    expect(r.byDistance.over3.steps).toBeGreaterThan(0);
+    expect(r.byDistance.over3.right).toBeLessThan(r.byDistance.over3.steps);
+  });
+
+  it("cannot judge on fewer than ten borewells", () => {
+    const sites = town(3, 0.5);
+    sites[4] = site("upside-down", sites[4].borewell.latitude, sites[4].borewell.longitude, SAND_OVER_CLAY);
+    const r = check(sites);
+    expect(r.compared).toBe(9);
+    expect(r.canJudge).toBe(false);
+    expect(r.oddOnesOut).toEqual([]);
+  });
+
+  it("does not divide by zero when nothing is comparable", () => {
+    // Twelve borewells, each more than 5 km from every other.
+    const r = check(Array.from({ length: 12 }, (_, i) => site(`far${i}`, north((Math.floor(i / 4) - 1) * 6), east(((i % 4) - 1.5) * 6), CLAY_OVER_SAND)));
+    expect(r.compared).toBe(0);
+    expect(r.canJudge).toBe(false);
+    expect(r.overall).toEqual({ steps: 0, right: 0 });
+    expect(share(r.overall)).toBeNull();
+    expect(r.oddOnesOut).toEqual([]);
+  });
+
+  it("puts borewells in the band of their nearest neighbour, and scores guesses the model would not show", () => {
+    // Three pairs, far from each other; the borewells of a pair are 0.5, 2 and 4 km apart.
+    const pair = (name: string, y: number, gap: number) => [site(`${name}a`, north(y), east(-10), CLAY_OVER_SAND), site(`${name}b`, north(y), east(-10 + gap), CLAY_OVER_SAND)];
+    const r = check([...pair("near", -12, 0.5), ...pair("mid", 0, 2), ...pair("far", 12, 4)]);
+    const both = { steps: 40, right: 40 };
+    expect(r.byDistance).toEqual({ under1: both, from1to3: both, over3: both });
+    // One borewell is one site, which the model never shows as an estimate; the check still scores it,
+    // but keeps it out of the headline.
+    expect(r.none).toEqual({ steps: 120, right: 120 });
+    expect(r.overall).toEqual({ steps: 0, right: 0 });
+  });
+
+  it("compares by height, using the hidden borewell's own ground", () => {
+    // Surveyed 10 m above its neighbours, and logged accordingly: the same layers by height.
+    const sites = town(4, 0.5);
+    const b = sites[5].borewell;
+    sites[5] = site("on-a-mound", b.latitude, b.longitude, [[0, 82.8, "clay"], [82.8, 132.8, "medium_sand"]], { groundElevationM: 130 });
+    const r = check(sites);
+    expect(r.overall.steps).toBe(320);
+    expect(r.overall.right).toBe(320);
+  });
+});