From 069177ba3c1f1e6af327430174b5d940ed461b6b Mon Sep 17 00:00:00 2001 From: galimba Date: Tue, 7 Jul 2026 16:56:29 +0200 Subject: [PATCH] feat(scripts): add consolidate command for overlapping stale pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New audit module audits/audit-consolidate.sh: vault-tools.sh consolidate finds groups of 3+ pages that are all past their staleness threshold and topically overlapping — pages pair when they share two or more approved tags plus a related: link, groups are the connected components of that graph. Writes candidates to memory/notes/consolidation-YYYY-MM-DD.md (valid type: report frontmatter) with members, shared tags, link evidence, and suggested action. Report-only: never merges, deletes, or modifies wiki pages. Consolidation reports join lint reports in .gitignore. Adds tests/test-consolidate.sh. Closes #16 Co-Authored-By: Claude Fable 5 --- .gitignore | 6 +- .vault/scripts/audits/audit-consolidate.sh | 301 +++++++++++++++++++++ .vault/scripts/tests/test-consolidate.sh | 149 ++++++++++ .vault/scripts/vault-tools.sh | 3 + CHANGELOG.md | 8 + 5 files changed, 465 insertions(+), 2 deletions(-) create mode 100644 .vault/scripts/audits/audit-consolidate.sh create mode 100755 .vault/scripts/tests/test-consolidate.sh diff --git a/.gitignore b/.gitignore index 0598803..db57332 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ tasks/ # Logs *.log -# Lint reports are regenerated by `vault-tools.sh lint --report` / `doctor`. -# Keep them out of version control so each clone computes a fresh one. +# Lint and consolidation reports are regenerated by `vault-tools.sh` +# (`lint --report` / `doctor` / `consolidate`). Keep them out of version +# control so each clone computes fresh ones. memory/notes/lint-report-*.md +memory/notes/consolidation-*.md diff --git a/.vault/scripts/audits/audit-consolidate.sh b/.vault/scripts/audits/audit-consolidate.sh new file mode 100644 index 0000000..b1c5149 --- /dev/null +++ b/.vault/scripts/audits/audit-consolidate.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# ============================================================================== +# AUDIT: Consolidation Candidates +# ============================================================================== +# Purpose: Identify groups of 3+ stale, topically overlapping wiki pages that +# are candidates for human-driven merging (issue #16). +# Usage: vault-tools.sh consolidate +# Dependencies: Requires lib-utils.sh to be sourced first. +# +# PAIRING RULE (deterministic, no tuning knobs): +# Two pages PAIR when they share >= 2 approved tags AND at least one of +# them lists the other in its `related:` frontmatter. The DIRECT-REFERENCE +# variant is implemented: a mutual or one-way `related:` entry pairs the +# pages; merely sharing a common third `related:` target does NOT. +# +# GROUPING RULE: +# Only pages already past their staleness threshold enter the pair graph +# (resolve_stale_threshold per page; statuses matching is_stale_exempt — +# archived/deprecated — are skipped). Candidate groups are the connected +# components of the pair graph with >= 3 members, so every member of every +# reported group is past its own threshold by construction. +# +# REPORT-ONLY: writes memory/notes/consolidation-YYYY-MM-DD.md and never +# modifies, merges, or deletes any wiki page. Exits 0 (advisory) unless +# report I/O fails. +# ============================================================================== + +# This file is sourced by vault-tools.sh — do not execute directly +[[ "${BASH_SOURCE[0]}" == "${0}" ]] && { echo "Source this file, don't execute it directly."; exit 1; } + +# Extract `related:` wikilink targets from a frontmatter blob, one basename +# per line. Handles quoted entries, [[path]] brackets, and |display suffixes. +# Usage: _consolidate_related_basenames "$frontmatter" +_consolidate_related_basenames() { + local fm="$1" + local in_rel=false + echo "$fm" | while IFS= read -r line; do + if [[ "$line" =~ ^related: ]]; then + in_rel=true + continue + fi + if $in_rel; then + if [[ "$line" =~ ^[[:space:]]*-[[:space:]] ]]; then + local entry + entry=$(echo "$line" \ + | sed 's/^[[:space:]]*-[[:space:]]*//' \ + | sed 's/[[:space:]]*$//' \ + | sed 's/^["'"'"']//' | sed 's/["'"'"']$//' \ + | sed 's/^\[\[//' | sed 's/\]\]$//' | sed 's/|.*$//') + [[ -n "$entry" ]] && basename "$entry" + elif [[ "$line" =~ ^[a-zA-Z] ]]; then + break + fi + fi + done +} + +# Union-find over the global _CONS_PARENT array. +_consolidate_find() { + local x=$1 + while [[ ${_CONS_PARENT[$x]} -ne $x ]]; do + x=${_CONS_PARENT[$x]} + done + echo "$x" +} + +cmd_consolidate() { + header "Consolidation Candidates" + + local today today_ts + today=$(date +%Y-%m-%d) + today_ts=$(date +%s) + + # Load approved tag taxonomy (same extraction as tag-audit). Only + # approved tags count toward the >= 2 shared-tag pairing rule. + local approved_file + approved_file=$(mktemp) + if [[ -f "${TAGS_FILE}" ]]; then + grep -oE '`[a-z][a-z0-9-]*/[a-z][a-z0-9-]*`' "${TAGS_FILE}" \ + | sed 's/`//g' | sort -u > "$approved_file" + else + warning "Tags file not found — treating all tags as approved" + fi + + # ------------------------------------------------------------------ + # Collect stale candidate pages + # ------------------------------------------------------------------ + subheader "Collecting stale candidate pages" + local -a c_file=() c_title=() c_tags=() c_related=() c_updated=() c_age=() c_threshold=() + while IFS= read -r file; do + local fm + fm=$(extract_fm "$file") + [[ -z "$fm" ]] && continue + + local status + status=$(fm_field "status" "$fm") + if is_stale_exempt "$status"; then + continue + fi + + local updated + updated=$(fm_field "updated" "$fm") + [[ -z "$updated" ]] && continue + [[ "$updated" == *"{{"* ]] && continue + if ! is_valid_date "$updated"; then + continue + fi + + local updated_ts + updated_ts=$(date -d "$updated" +%s 2>/dev/null || date -j -f "%Y-%m-%d" "$updated" +%s 2>/dev/null || echo "0") + [[ "$updated_ts" == "0" ]] && continue + + local file_threshold age_days + file_threshold=$(resolve_stale_threshold "$fm") + age_days=$(( (today_ts - updated_ts) / 86400 )) + if [[ $age_days -lt $file_threshold ]]; then + continue + fi + + local tags + tags=$(fm_tags "$fm" | sort -u) + if [[ -s "$approved_file" ]]; then + tags=$(comm -12 <(echo "$tags") "$approved_file") + fi + + c_file+=("${file#"${VAULT_ROOT}"/}") + c_title+=("$(fm_field "title" "$fm")") + c_tags+=("$tags") + c_related+=("$(_consolidate_related_basenames "$fm")") + c_updated+=("$updated") + c_age+=("$age_days") + c_threshold+=("$file_threshold") + done < <(wiki_files) + + local n=${#c_file[@]} + ok "${n} stale candidate page(s)" + + # ------------------------------------------------------------------ + # Build pair graph: >= 2 shared approved tags AND a direct related: + # reference (mutual or one-way). Group via union-find. + # ------------------------------------------------------------------ + subheader "Building pair graph" + _CONS_PARENT=() + local -a pair_i=() pair_j=() pair_text=() + local i j + for ((i = 0; i < n; i++)); do + _CONS_PARENT[i]=$i + done + for ((i = 0; i < n; i++)); do + for ((j = i + 1; j < n; j++)); do + local shared shared_count + shared=$(comm -12 <(echo "${c_tags[$i]}") <(echo "${c_tags[$j]}") | grep -c . || true) + shared_count=${shared:-0} + [[ $shared_count -lt 2 ]] && continue + + local base_i base_j i_to_j=false j_to_i=false + base_i=$(basename "${c_file[$i]}") + base_j=$(basename "${c_file[$j]}") + grep -Fxq "$base_j" <<< "${c_related[$i]}" && i_to_j=true + grep -Fxq "$base_i" <<< "${c_related[$j]}" && j_to_i=true + if ! $i_to_j && ! $j_to_i; then + continue + fi + + local direction + if $i_to_j && $j_to_i; then + direction="mutual \`related:\`" + elif $i_to_j; then + direction="\`related:\` from ${base_i}" + else + direction="\`related:\` from ${base_j}" + fi + local shared_list + shared_list=$(comm -12 <(echo "${c_tags[$i]}") <(echo "${c_tags[$j]}") | sed 's/^/`/; s/$/`/' | paste -sd',' - | sed 's/,/, /g') + + pair_i+=("$i") + pair_j+=("$j") + pair_text+=("[[${c_file[$i]}]] and [[${c_file[$j]}]] — ${direction}; ${shared_count} shared tags: ${shared_list}") + + # Union + local ri rj + ri=$(_consolidate_find "$i") + rj=$(_consolidate_find "$j") + [[ $ri -ne $rj ]] && _CONS_PARENT[rj]=$ri + done + done + ok "${#pair_i[@]} qualifying pair(s)" + + # Connected components with >= 3 members + local -A comp_members=() + for ((i = 0; i < n; i++)); do + local root + root=$(_consolidate_find "$i") + comp_members[$root]="${comp_members[$root]:-} $i" + done + local -a groups=() + local root + for root in "${!comp_members[@]}"; do + read -r -a members <<< "${comp_members[$root]}" + [[ ${#members[@]} -ge 3 ]] && groups+=("${comp_members[$root]}") + done + + # ------------------------------------------------------------------ + # Write report (report-only: no wiki page is ever modified here) + # ------------------------------------------------------------------ + local notes_dir="${MEMORY_DIR}/notes" + local report_file="${notes_dir}/consolidation-${today}.md" + mkdir -p "$notes_dir" + + cat > "$report_file" <= 2 approved tags AND one lists the other in its +\`related:\` frontmatter (direct reference — mutual or one-way; a shared +third-party \`related:\` target does not pair). Groups are connected +components of the pair graph with >= 3 members; every member is past its +staleness threshold. Report-only: no wiki page was modified. + +## Summary + +- Stale candidate pages: ${n} +- Qualifying pairs: ${#pair_i[@]} +- Consolidation groups: ${#groups[@]} +EOF + + if [[ ${#groups[@]} -eq 0 ]]; then + { + echo "" + echo "## Groups" + echo "" + echo "_No consolidation candidates found. No action needed._" + } >> "$report_file" + else + local g=0 m p + for root in "${!comp_members[@]}"; do + read -r -a members <<< "${comp_members[$root]}" + [[ ${#members[@]} -lt 3 ]] && continue + g=$((g + 1)) + + # Group-wide tag intersection + local group_tags="${c_tags[${members[0]}]}" + for m in "${members[@]}"; do + group_tags=$(comm -12 <(echo "$group_tags") <(echo "${c_tags[$m]}")) + done + local group_tags_line + group_tags_line=$(echo "$group_tags" | sed '/^$/d' | sed 's/^/`/; s/$/`/' | paste -sd',' - | sed 's/,/, /g') + [[ -z "$group_tags_line" ]] && group_tags_line="_(none shared by all members; overlap is pairwise)_" + + { + echo "" + echo "## Group ${g} (${#members[@]} pages)" + echo "" + echo "### Members" + echo "" + for m in $(printf '%s\n' "${members[@]}" | sort -n); do + echo "- [[${c_file[$m]}]] — \"${c_title[$m]}\" (updated ${c_updated[$m]}, ${c_age[$m]} days old, threshold ${c_threshold[$m]})" + done + echo "" + echo "### Shared tags (all members)" + echo "" + echo "- ${group_tags_line}" + echo "" + echo "### Link evidence" + echo "" + for ((p = 0; p < ${#pair_i[@]}; p++)); do + if [[ " ${members[*]} " == *" ${pair_i[$p]} "* && " ${members[*]} " == *" ${pair_j[$p]} "* ]]; then + echo "- ${pair_text[$p]}" + fi + done + echo "" + echo "**Suggested action**: review these ${#members[@]} pages for merging into a single canonical page; set \`status: archived\` on superseded members." + } >> "$report_file" + done + fi + + rm -f "$approved_file" + + subheader "Results" + if [[ ${#groups[@]} -eq 0 ]]; then + ok "No consolidation candidates found" + else + warning "${#groups[@]} consolidation group(s) found — human review suggested" + fi + ok "Consolidation report written to memory/notes/consolidation-${today}.md" + echo "" +} diff --git a/.vault/scripts/tests/test-consolidate.sh b/.vault/scripts/tests/test-consolidate.sh new file mode 100755 index 0000000..7888f78 --- /dev/null +++ b/.vault/scripts/tests/test-consolidate.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Test for `vault-tools.sh consolidate` (issue #16). +# +# Seeds a temp vault with three mutually-related concept pages that share +# 2+ approved tags and are past their staleness threshold, plus one fresh +# page with the same tags and links. Asserts: +# - consolidate exits 0 and writes memory/notes/consolidation-YYYY-MM-DD.md +# - all three stale pages land in one group +# - the fresh page is NOT in any group +# - the report has valid frontmatter (title/type/created) +# +# Run: bash .vault/scripts/tests/test-consolidate.sh +# Exit: 0 on PASS, non-zero on any FAIL. + +set -euo pipefail + +VAULT_ROOT="$(git rev-parse --show-toplevel)" +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +TODAY="$(date +%Y-%m-%d)" +# ~60 days back: past the 30-day threshold for domain/engineering pages. +STALE_DATE="$(date -d "-60 days" +%Y-%m-%d)" + +# ------------------------------------------------------------------ +# Build a minimal vault clone in a tempdir. vault-tools.sh resolves +# VAULT_ROOT from its own location, so running the copied script +# operates entirely inside the tempdir — no git repo needed. +# ------------------------------------------------------------------ +mkdir -p "$TMPDIR/wiki/concepts" "$TMPDIR/memory/notes" +cp -r "$VAULT_ROOT/.vault" "$TMPDIR/.vault" + +cat > "$TMPDIR/wiki/index.md" < "$TMPDIR/wiki/log.md" < b, b -> c, c -> a), sharing +# the approved tags domain/engineering + type/concept + lifecycle/active. +seed_page() { + local name="$1" related="$2" updated="$3" + cat > "$TMPDIR/wiki/concepts/concept-${name}.md" </dev/null) \ + || fail "consolidate exited non-zero" + +report="$TMPDIR/memory/notes/consolidation-${TODAY}.md" +[[ -f "$report" ]] || fail "report not written to memory/notes/consolidation-${TODAY}.md" + +# ------------------------------------------------------------------ +# Assert 1: report has valid frontmatter fields. +# ------------------------------------------------------------------ +grep -q "^title: \"Consolidation Report ${TODAY}\"" "$report" || fail "report missing title frontmatter" +grep -q "^type: report" "$report" || fail "report missing type: report" +grep -q "^created: ${TODAY}" "$report" || fail "report missing created: ${TODAY}" +grep -q "^ - type/report" "$report" || fail "report missing type/report tag" +grep -q "^ - lifecycle/active" "$report" || fail "report missing lifecycle/active tag" + +# ------------------------------------------------------------------ +# Assert 2: exactly one group containing all three stale pages. +# ------------------------------------------------------------------ +grep -q "^## Group 1" "$report" || fail "report has no Group 1" +if grep -q "^## Group 2" "$report"; then + fail "report has more than one group" +fi +group_section="$(awk '/^## Group 1/,0' "$report")" +for name in alpha beta gamma; do + grep -q "concept-${name}.md" <<< "$group_section" \ + || fail "concept-${name}.md missing from Group 1" +done + +# ------------------------------------------------------------------ +# Assert 3: the fresh page is not in any group. +# ------------------------------------------------------------------ +if grep -q "concept-delta.md" <<< "$group_section"; then + fail "fresh page concept-delta.md appeared in a group" +fi + +# ------------------------------------------------------------------ +# Assert 4: report-only — no wiki page was modified. +# ------------------------------------------------------------------ +grep -q "updated: ${STALE_DATE}" "$TMPDIR/wiki/concepts/concept-alpha.md" \ + || fail "consolidate modified a wiki page" + +echo "PASS: consolidate groups stale overlapping pages and writes the report" diff --git a/.vault/scripts/vault-tools.sh b/.vault/scripts/vault-tools.sh index b7ad924..d39447a 100644 --- a/.vault/scripts/vault-tools.sh +++ b/.vault/scripts/vault-tools.sh @@ -18,6 +18,7 @@ # ./vault-tools.sh skill-audit Audit skills against hardening policy # ./vault-tools.sh skill-manifest Generate/refresh skill-manifest.json # ./vault-tools.sh content-audit Audit content integrity +# ./vault-tools.sh consolidate Report stale overlapping pages to merge # ./vault-tools.sh stats Show vault statistics # ./vault-tools.sh init-hooks Install git hooks # ./vault-tools.sh doctor Full diagnostic check @@ -110,6 +111,7 @@ cmd_help() { echo " skill-audit Audit skill security" echo " skill-manifest Generate or refresh a skill's manifest" echo " content-audit Audit content integrity" + echo " consolidate Report groups of stale overlapping pages to merge" echo "" echo "Management:" echo " status Show vault status" @@ -140,6 +142,7 @@ main() { skill-audit) cmd_skill_audit "$@" ;; skill-manifest) cmd_skill_manifest "$@" ;; content-audit) cmd_content_audit "$@" ;; + consolidate) cmd_consolidate "$@" ;; index-rebuild) cmd_index_rebuild "$@" ;; init-hooks) cmd_init_hooks "$@" ;; doctor) cmd_doctor "$@" ;; diff --git a/CHANGELOG.md b/CHANGELOG.md index 635b357..29a06d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `vault-tools.sh consolidate` — report-only command that finds groups of + 3+ stale, overlapping wiki pages (pairs share >= 2 approved tags and a + direct `related:` reference; groups are connected components where every + member is past its staleness threshold) and writes candidates to + `memory/notes/consolidation-YYYY-MM-DD.md` for human-driven merging (#16). + ## [0.5.0] - 2026-07-07 ### Added