From fe91b051d81917c70259be494e4e9e817c2d5aa7 Mon Sep 17 00:00:00 2001
From: Nils Lehnen <30603423+iderex@users.noreply.github.com>
Date: Thu, 13 Aug 2026 19:06:49 +0200
Subject: [PATCH] Produce the two files a crawler asks for, and refuse a
sitemap that drifts [#69]
A crawler asks for the exclusion file before it asks for anything else and a
sitemap is how it is told which addresses exist. The build wrote neither, so
both requests were answered with the not-found page: a request the reader's
host serves for nothing, and an answer carrying none of what was asked for.
The build now writes robots.txt and sitemap.xml, last, after everything that
can put a page into the output. The sitemap lists every produced page at the
address it is served at, sorted, and leaves out the not-found page, which is
served in answer to addresses that are not its own and would otherwise be
offered for indexing under an address the site claims to have. Neither file
carries a date, because a sitemap stating when each page last changed would
produce different bytes from unchanged source on two days.
The failure the new gate leg prevents is the day after. A generated file agrees
with the output on the day it is written, so what reds here is a page written
by something added below the writer and listed nowhere, or an address left in
the list with no page behind it. The first is invisible to every other check:
the page is valid, it is served, and nothing tells a crawler it exists. The
second sends every client that reads the file to the not-found page. The leg
walks the output directory rather than the list the build reported, so what it
compares is the file on disk against the files on disk.
Which pages a sitemap carries is one function, used by the writer and by the
comparison, because a rule stated in both places would agree on the day the
second statement was written and never be checked against it again.
The suite cases that asserted how many files a build reports now ask whether it
reported the file they are about. Counting was a statement about the whole set,
which every new writer moves and none of those cases was about.
The icon that issue also asks for is not here. What it shows is entry 7 of #7
and is not settled, so it waits rather than being answered by whatever this
change would have drawn.
Signed-off-by: Nils Lehnen <30603423+iderex@users.noreply.github.com>
---
CHANGELOG.md | 7 ++
internal/gate/gate.go | 20 ++++
internal/reproduce/reproduce_test.go | 8 +-
internal/site/crawler.go | 146 +++++++++++++++++++++++++
internal/site/crawler_test.go | 142 ++++++++++++++++++++++++
internal/site/privacy_test.go | 6 +-
internal/site/site.go | 18 +++
internal/site/site_test.go | 27 ++++-
internal/sitemap/sitemap.go | 158 +++++++++++++++++++++++++++
internal/sitemap/sitemap_test.go | 124 +++++++++++++++++++++
10 files changed, 646 insertions(+), 10 deletions(-)
create mode 100644 internal/site/crawler.go
create mode 100644 internal/site/crawler_test.go
create mode 100644 internal/sitemap/sitemap.go
create mode 100644 internal/sitemap/sitemap_test.go
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 242f120..7a58cbb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,3 +64,10 @@ there is no earlier bundle to compare it against.
it, packed so that two runs of one source produce one archive.
- The bill of materials states the version of the thing it is about, so two
archives are distinguishable by the document each one carries.
+- The build writes `robots.txt` and `sitemap.xml`. Both are asked for by clients
+ without any page linking them, and a bundle without them answers those
+ requests with the not-found page. The sitemap lists every page the build wrote
+ except the not-found one, at the address each is served at, and carries no
+ date, so two builds of one source still produce one set of bytes.
+- The gate refuses a bundle whose sitemap disagrees with the pages beside it, in
+ both directions: a page listed nowhere, and an address with no page behind it.
diff --git a/internal/gate/gate.go b/internal/gate/gate.go
index c2ef990..6db1266 100644
--- a/internal/gate/gate.go
+++ b/internal/gate/gate.go
@@ -21,6 +21,7 @@ import (
"github.com/Flowfin/site/internal/invariant"
"github.com/Flowfin/site/internal/link"
"github.com/Flowfin/site/internal/site"
+ "github.com/Flowfin/site/internal/sitemap"
)
// A leg is one thing the gate decides. Its report is the sentence printed
@@ -38,6 +39,7 @@ func legs() []leg {
{"test", testLeg},
{"build", buildLeg},
{"links", linksLeg},
+ {"sitemap", sitemapLeg},
{"invariants", invariantsLeg},
}
}
@@ -209,6 +211,24 @@ func linksLeg(root string) (string, error) {
return strings.TrimSpace(lines[len(lines)-1]), nil
}
+// sitemapLeg refuses a sitemap that disagrees with the pages beside it. It sits
+// after the links leg because both walk what the build produced and the links
+// leg answers the more basic question: a site whose pages point at files nobody
+// wrote is broken for a reader, where a sitemap that has drifted is broken only
+// for a crawler.
+func sitemapLeg(root string) (string, error) {
+ var log strings.Builder
+ if err := sitemap.Run(root, &log); err != nil {
+ lines := strings.Split(strings.TrimRight(log.String(), "\n"), "\n")
+ return "", fmt.Errorf("%v:\n%s", err, indent(lines))
+ }
+ // The last line the comparison wrote is the one that says what it
+ // covered, in its own words, so a run that examined nothing says so here
+ // rather than reporting a count assembled in this file.
+ lines := strings.Split(strings.TrimRight(log.String(), "\n"), "\n")
+ return strings.TrimSpace(lines[len(lines)-1]), nil
+}
+
// invariantsLeg decides the rules that can be read off the tree and off the
// output a build produces. The rows live in their own package rather than here,
// so the same set is decided by this leg and by the workflow that reports it
diff --git a/internal/reproduce/reproduce_test.go b/internal/reproduce/reproduce_test.go
index 447b56c..7b76d92 100644
--- a/internal/reproduce/reproduce_test.go
+++ b/internal/reproduce/reproduce_test.go
@@ -54,12 +54,18 @@ func write(t *testing.T, root, name, body string) {
// The neighbour, and it is the real build rather than a stand-in: a tree whose
// build reads only what is committed produces the same bytes twice, and the run
// says how many files it compared rather than passing silently.
+//
+// The count is the page and the two files a crawler asks for. It is written out
+// rather than derived, because those two are the ones a build could most easily
+// make unreproducible: a sitemap may state when each page last changed, and one
+// carrying today's date would pass every other check in this tree and red only
+// here.
func TestTwoBuildsOfTheRealGeneratorAgree(t *testing.T) {
var log bytes.Buffer
if err := Run(tree(t), &log); err != nil {
t.Fatalf("Run refused two builds of one source: %v\n%s", err, log.String())
}
- if !strings.Contains(log.String(), "1 file(s), identical in both builds") {
+ if !strings.Contains(log.String(), "3 file(s), identical in both builds") {
t.Errorf("the run did not say what it compared; it said:\n%s", log.String())
}
}
diff --git a/internal/site/crawler.go b/internal/site/crawler.go
new file mode 100644
index 0000000..a0d4863
--- /dev/null
+++ b/internal/site/crawler.go
@@ -0,0 +1,146 @@
+// The files a crawler asks for without any page linking them, and what the
+// build writes into them.
+//
+// A crawler asks for the exclusion file before it asks for anything else, and it
+// asks again on every site it has not seen answer. Where the repository provides
+// none, the host answers with the not-found page: a request the reader's host
+// serves for nothing, and an answer that carries none of what was asked for.
+// Producing the file is what turns that into one short answer.
+//
+// The sitemap is generated rather than written down. Twelve of the addresses
+// this site will have come from a file somebody else edits, so a hand-written
+// list is wrong the day a row is added and nothing about the wrong list looks
+// wrong. What the generated one lists is what this build just wrote.
+//
+// Neither file carries a date. A sitemap may state when each page last changed,
+// and a build that wrote today's date into one would produce different bytes
+// from the same source on two days, against a check that exists to compare
+// exactly that. What it costs is that a crawler is told which addresses exist
+// and not which of them moved, and the second half is worth less here than a
+// build somebody can reproduce.
+package site
+
+import (
+ "encoding/xml"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strings"
+)
+
+// RobotsPath and SitemapPath are where the two files land. Neither name is this
+// repository's choice: the first is the one address a crawler asks for before it
+// has read anything, and the second is found only because the first names it.
+const (
+ RobotsPath = "robots.txt"
+ SitemapPath = "sitemap.xml"
+)
+
+// SitemapAddresses is which of the produced files a sitemap lists, and at which
+// address. It takes paths relative to the output directory.
+//
+// It is exported because two readers need the same answer: the writer below,
+// which turns it into the file, and the leg that walks the output afterwards and
+// compares the file against it. Stating the rule in both places would state it
+// twice, and the second statement would agree with the first on the day it was
+// written and never be checked against it again.
+//
+// The not-found page is the one page left out, and it is left out rather than
+// forgotten. A sitemap is a list of addresses a crawler is invited to fetch and
+// index, and that page is served in answer to addresses that are not its own, so
+// listing it asks for an index entry that sends a reader to an error page under
+// an address the site says it has. Every other produced page is listed, and
+// anything that is not a page is not, because what a sitemap carries is the
+// addresses a reader can be sent to.
+//
+// The addresses are sorted, so the file the build writes is a property of which
+// pages exist rather than of the order the writers happen to run in.
+func SitemapAddresses(produced []string) []string {
+ var addresses []string
+ for _, name := range produced {
+ name = strings.TrimPrefix(path.Clean(filepath.ToSlash(name)), "/")
+ if !strings.HasSuffix(name, ".html") || name == NotFoundPath {
+ continue
+ }
+ addresses = append(addresses, Origin+addressOf(name))
+ }
+ sort.Strings(addresses)
+ return addresses
+}
+
+// writeSitemap writes the list of addresses this build produced, and reports
+// what it wrote.
+//
+// A build that produced no page writes no sitemap and says so, rather than
+// writing a list of nothing. An empty list is a file that says this site has no
+// pages, which is a statement about the site rather than a statement that the
+// build had nothing to say.
+func writeSitemap(out, label string, written []string, log io.Writer) ([]string, error) {
+ addresses := SitemapAddresses(relativeTo(label, written))
+ if len(addresses) == 0 {
+ fmt.Fprintf(log, "the build produced no page, so no %s was written\n", SitemapPath)
+ return nil, nil
+ }
+
+ var body strings.Builder
+ body.WriteString(xml.Header)
+ body.WriteString(`` + "\n")
+ for _, a := range addresses {
+ var escaped strings.Builder
+ if err := xml.EscapeText(&escaped, []byte(a)); err != nil {
+ return nil, fmt.Errorf("writing %s: %w", SitemapPath, err)
+ }
+ fmt.Fprintf(&body, " %s\n", escaped.String())
+ }
+ body.WriteString("\n")
+
+ name := filepath.Join(out, filepath.FromSlash(SitemapPath))
+ if err := os.WriteFile(name, []byte(body.String()), 0o644); err != nil {
+ return nil, fmt.Errorf("writing %s: %w", SitemapPath, err)
+ }
+ slashed := path.Join(label, SitemapPath)
+ fmt.Fprintf(log, "wrote %s (%d bytes, %d address(es) listed)\n", slashed, body.Len(), len(addresses))
+ return []string{slashed}, nil
+}
+
+// writeRobots writes the exclusion file. Nothing on this site is kept out of an
+// index, so the file says that plainly and spends the rest of itself naming
+// where the list of addresses is.
+//
+// It names the sitemap only where one was written. A robots file pointing at an
+// address the build did not produce sends the one client that reads it to the
+// not-found page, which is the failure this pair exists to remove, arriving from
+// the file that was supposed to remove it.
+func writeRobots(out, label string, sitemap []string, log io.Writer) ([]string, error) {
+ var body strings.Builder
+ body.WriteString("# Nothing on this site is kept out of an index. This file exists so that a\n")
+ body.WriteString("# crawler asking for it is answered rather than served the not-found page.\n")
+ body.WriteString("User-agent: *\n")
+ body.WriteString("Disallow:\n")
+ if len(sitemap) > 0 {
+ fmt.Fprintf(&body, "\nSitemap: %s/%s\n", Origin, SitemapPath)
+ }
+
+ name := filepath.Join(out, filepath.FromSlash(RobotsPath))
+ if err := os.WriteFile(name, []byte(body.String()), 0o644); err != nil {
+ return nil, fmt.Errorf("writing %s: %w", RobotsPath, err)
+ }
+ slashed := path.Join(label, RobotsPath)
+ fmt.Fprintf(log, "wrote %s (%d bytes, %d sitemap(s) named)\n", slashed, body.Len(), len(sitemap))
+ return []string{slashed}, nil
+}
+
+// relativeTo strips the output label off the paths the writers report, so what
+// this file works in is the addresses the host will serve rather than wherever
+// the run happened to render.
+func relativeTo(label string, written []string) []string {
+ prefix := filepath.ToSlash(label) + "/"
+ out := make([]string, 0, len(written))
+ for _, w := range written {
+ out = append(out, strings.TrimPrefix(filepath.ToSlash(w), prefix))
+ }
+ return out
+}
diff --git a/internal/site/crawler_test.go b/internal/site/crawler_test.go
new file mode 100644
index 0000000..d28a0ed
--- /dev/null
+++ b/internal/site/crawler_test.go
@@ -0,0 +1,142 @@
+// The suite over the two files a crawler asks for.
+//
+// The cases about which addresses belong in a sitemap drive the rule directly
+// rather than through a build, because the rule is what the leg over the output
+// reads and a case that went through a build would be testing the build.
+package site
+
+import (
+ "io"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// The not-found page is produced, is a page, and is the one page a sitemap must
+// not carry. Listing it asks a crawler to index the document a host serves for
+// addresses this site does not have, under an address the site says it has.
+func TestSitemapLeavesOutTheNotFoundPage(t *testing.T) {
+ produced := []string{"index.html", "privacy/index.html", NotFoundPath}
+
+ got := SitemapAddresses(produced)
+
+ for _, a := range got {
+ if strings.HasSuffix(a, "/"+NotFoundPath) {
+ t.Fatalf("the sitemap lists the not-found page: %v", got)
+ }
+ }
+ if len(got) != 2 {
+ t.Fatalf("SitemapAddresses(%v) = %v, want the two pages that are not the not-found one", produced, got)
+ }
+}
+
+// A directory address is served by the index document inside it, and the
+// sitemap states the address rather than the file. A crawler told to fetch the
+// file would index a second address for a page that already has one.
+func TestSitemapStatesTheAddressAndNotTheFile(t *testing.T) {
+ got := SitemapAddresses([]string{"index.html", "privacy/index.html"})
+
+ want := []string{Origin + "/", Origin + "/privacy/"}
+ if len(got) != len(want) {
+ t.Fatalf("SitemapAddresses gave %v, want %v", got, want)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Errorf("entry %d is %s, want %s", i, got[i], want[i])
+ }
+ }
+}
+
+// What a sitemap carries is addresses a reader can be sent to. The reporting
+// route, the exclusion file and the sitemap itself are all produced and none of
+// them is a page.
+func TestSitemapCarriesOnlyPages(t *testing.T) {
+ produced := []string{
+ "index.html",
+ ".well-known/security.txt",
+ RobotsPath,
+ SitemapPath,
+ "nested/style.css",
+ }
+
+ got := SitemapAddresses(produced)
+
+ if len(got) != 1 || got[0] != Origin+"/" {
+ t.Fatalf("SitemapAddresses(%v) = %v, want the one page", produced, got)
+ }
+}
+
+// The order of the file is a property of which pages exist. Two runs over the
+// same tree produce the same bytes, which is what the check that builds twice
+// compares, and the order the writers happen to run in is not part of it.
+func TestSitemapIsSortedWhateverOrderThePagesArrivedIn(t *testing.T) {
+ first := SitemapAddresses([]string{"privacy/index.html", "index.html", "legal/index.html"})
+ second := SitemapAddresses([]string{"index.html", "legal/index.html", "privacy/index.html"})
+
+ if len(first) != len(second) {
+ t.Fatalf("the two orders gave %v and %v", first, second)
+ }
+ for i := range first {
+ if first[i] != second[i] {
+ t.Fatalf("the two orders gave %v and %v", first, second)
+ }
+ }
+ if first[0] != Origin+"/" {
+ t.Errorf("the list opens with %s, and sorted it opens with the site root", first[0])
+ }
+}
+
+// A build that produced no page writes no sitemap and says so. An empty list is
+// a file stating that this site has no pages, which is a claim rather than a
+// report that there was nothing to write.
+func TestNoPageMeansNoSitemapAndASaidReason(t *testing.T) {
+ out := t.TempDir()
+
+ var log strings.Builder
+ written, err := writeSitemap(out, "dist", []string{"dist/" + RobotsPath}, &log)
+ if err != nil {
+ t.Fatalf("writeSitemap: %v", err)
+ }
+ if len(written) != 0 {
+ t.Errorf("writeSitemap reported %v out of a build with no page", written)
+ }
+ if !strings.Contains(log.String(), "produced no page") {
+ t.Errorf("the run does not say why nothing was written:\n%s", log.String())
+ }
+}
+
+// The exclusion file names the sitemap only where one was written. A robots file
+// pointing at an address the build did not produce sends the one client that
+// reads it to the not-found page, which is the failure this pair exists to
+// remove.
+func TestRobotsNamesNoSitemapThatWasNotWritten(t *testing.T) {
+ out := t.TempDir()
+
+ if _, err := writeRobots(out, "dist", nil, io.Discard); err != nil {
+ t.Fatalf("writeRobots: %v", err)
+ }
+ got := read(t, filepath.Join(out, RobotsPath))
+
+ if strings.Contains(got, "Sitemap:") {
+ t.Errorf("the file names a sitemap the build did not write:\n%s", got)
+ }
+ if !strings.Contains(got, "User-agent: *") {
+ t.Errorf("the file excludes nobody and does not say so:\n%s", got)
+ }
+}
+
+// With a sitemap beside it the file names it, at the address a crawler will ask
+// for rather than at the path the build wrote.
+func TestRobotsNamesTheSitemapThatWasWritten(t *testing.T) {
+ out := t.TempDir()
+
+ if _, err := writeRobots(out, "dist", []string{"dist/" + SitemapPath}, io.Discard); err != nil {
+ t.Fatalf("writeRobots: %v", err)
+ }
+ got := read(t, filepath.Join(out, RobotsPath))
+
+ want := "Sitemap: " + Origin + "/" + SitemapPath
+ if !strings.Contains(got, want) {
+ t.Errorf("the file does not carry %q; it is:\n%s", want, got)
+ }
+}
diff --git a/internal/site/privacy_test.go b/internal/site/privacy_test.go
index a641fd5..9d3b871 100644
--- a/internal/site/privacy_test.go
+++ b/internal/site/privacy_test.go
@@ -277,7 +277,7 @@ func TestBuildWritesThePrivacyPageAndSaysWhatIsOnIt(t *testing.T) {
if err != nil {
t.Fatalf("the build refused: %v\n%s", err, log.String())
}
- if len(written) != 2 || written[1] != "dist/privacy/index.html" {
+ if !wrote(written, "dist/privacy/index.html") {
t.Fatalf("the build wrote %q", written)
}
if !strings.Contains(log.String(), "1 checked, 0 promised, 1 residual") {
@@ -327,8 +327,8 @@ func TestBuildSaysWhenThereIsNoPrivacyProse(t *testing.T) {
if err != nil {
t.Fatalf("the build refused: %v\n%s", err, log.String())
}
- if len(written) != 1 {
- t.Fatalf("the build wrote %q", written)
+ if wrote(written, "dist/privacy/index.html") {
+ t.Fatalf("the build wrote a privacy page out of a tree that carries no prose for one: %q", written)
}
if !strings.Contains(log.String(), "no content/privacy.txt in the tree") {
t.Errorf("the run passed over the absence:\n%s", log.String())
diff --git a/internal/site/site.go b/internal/site/site.go
index ba58be8..4473d64 100644
--- a/internal/site/site.go
+++ b/internal/site/site.go
@@ -186,6 +186,24 @@ func Build(root, outDir string, log io.Writer) ([]string, error) {
}
written = append(written, copied...)
+ // The two files nothing links are written last, after everything that can
+ // put a page into the output, because the sitemap is a list of what is
+ // above it and anything landing underneath it would be served and listed
+ // nowhere. That the ordering holds is not left to this comment: the leg
+ // over the output walks the directory afterwards and compares what the
+ // file lists against what is beside it.
+ sitemap, err := writeSitemap(out, label, written, log)
+ if err != nil {
+ return nil, err
+ }
+ written = append(written, sitemap...)
+
+ robots, err := writeRobots(out, label, sitemap, log)
+ if err != nil {
+ return nil, err
+ }
+ written = append(written, robots...)
+
fmt.Fprintf(log, "%d file(s) written into %s\n", len(written), label)
return written, nil
}
diff --git a/internal/site/site_test.go b/internal/site/site_test.go
index 12e5381..bad1d7a 100644
--- a/internal/site/site_test.go
+++ b/internal/site/site_test.go
@@ -40,6 +40,19 @@ func tree(t *testing.T, prose string) string {
return root
}
+// wrote answers whether the build reported a path. A case that asked instead
+// how many paths came back would be a case about the whole set of files the
+// build writes, which is a thing every new writer moves and no case here is
+// about.
+func wrote(written []string, want string) bool {
+ for _, w := range written {
+ if w == want {
+ return true
+ }
+ }
+ return false
+}
+
func mkdir(t *testing.T, dir string) {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
@@ -74,8 +87,8 @@ func TestBuildJoinsAWrappedParagraphIntoOneSentence(t *testing.T) {
if err != nil {
t.Fatalf("Build: %v", err)
}
- if len(written) != 1 || written[0] != "dist/index.html" {
- t.Fatalf("Build reported %v, want [dist/index.html]", written)
+ if !wrote(written, "dist/index.html") {
+ t.Fatalf("Build reported %v, and none of it is dist/index.html", written)
}
got := read(t, filepath.Join(root, OutputDir, "index.html"))
@@ -146,8 +159,10 @@ func TestBuildWritesAnAbsoluteOutputDirectoryWhereItWasAsked(t *testing.T) {
if _, err := os.Stat(filepath.Join(root, OutputDir)); !os.IsNotExist(err) {
t.Errorf("an absolute output directory still produced %s in the tree", filepath.Join(root, OutputDir))
}
- if len(written) != 1 {
- t.Errorf("Build reported %v, want one file", written)
+ for _, w := range written {
+ if !strings.HasPrefix(w, filepath.ToSlash(elsewhere)+"/") {
+ t.Errorf("Build reported %s, which is not under the directory it was asked for", w)
+ }
}
}
@@ -207,8 +222,8 @@ func TestBuildCopiesAssetsByteForByte(t *testing.T) {
if got := read(t, copied); got != body {
t.Errorf("the asset came out as %q, want %q", got, body)
}
- if len(written) != 2 || written[1] != "dist/nested/style.css" {
- t.Errorf("Build reported %v, want the page and dist/nested/style.css", written)
+ if !wrote(written, "dist/nested/style.css") {
+ t.Errorf("Build reported %v, and none of it is the copied asset", written)
}
}
diff --git a/internal/sitemap/sitemap.go b/internal/sitemap/sitemap.go
new file mode 100644
index 0000000..d44def1
--- /dev/null
+++ b/internal/sitemap/sitemap.go
@@ -0,0 +1,158 @@
+// Package sitemap refuses a sitemap that disagrees with the pages the build
+// wrote.
+//
+// The file is generated, so on the day it is written the two agree by
+// construction and a check over them proves nothing. What this refuses is the
+// day after. A page written by something that runs after the list is assembled,
+// or beside the flow that assembles it, lands in the output and is named
+// nowhere: the build is green, the page is served, and the only party who finds
+// out is a crawler that is never told to ask for it.
+//
+// The other direction is the one that is worse to serve. An address listed with
+// no page behind it sends every client that reads the list to the not-found
+// page, repeatedly, and the site is the party claiming the address exists.
+//
+// It walks the output rather than reading the list of paths the build reported,
+// so what it compares is the file on disk against the files on disk. A walk that
+// trusted the writer's own account of what it wrote would agree with the writer
+// about a page the writer never mentioned.
+package sitemap
+
+import (
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/Flowfin/site/internal/site"
+)
+
+// location is one address a sitemap states. The file is generated by this
+// repository and read by clients that accept far less than a parser would, so
+// this reads the one element it is about rather than the document around it: a
+// sitemap this build did not write is not a case that arises, and a whole XML
+// reader here would be apparatus nobody maintains.
+var location = regexp.MustCompile(`(?is)\s*([^<]*?)\s*`)
+
+// Listed returns the addresses a sitemap states, in the order it states them.
+func Listed(body []byte) []string {
+ var out []string
+ for _, m := range location.FindAllSubmatch(body, -1) {
+ out = append(out, string(m[1]))
+ }
+ return out
+}
+
+// Decide returns one detail per disagreement between what a sitemap lists and
+// what the produced pages are owed, in the order each side was given.
+//
+// Three things are refused and each names what it costs a reader, because the
+// two directions fail for opposite reasons and a failure saying only that the
+// two lists differ leaves the next person to work out which way round it is.
+func Decide(listed, owed []string) []string {
+ have := map[string]int{}
+ for _, l := range listed {
+ have[l]++
+ }
+ want := map[string]bool{}
+ for _, o := range owed {
+ want[o] = true
+ }
+
+ var details []string
+ for _, o := range owed {
+ if have[o] == 0 {
+ details = append(details, fmt.Sprintf(
+ "the build produced a page served at %s and %s lists no entry for it, so nothing ever tells a crawler the page is there",
+ o, site.SitemapPath))
+ }
+ }
+ reported := map[string]bool{}
+ for _, l := range listed {
+ switch {
+ case !want[l] && !reported[l]:
+ reported[l] = true
+ details = append(details, fmt.Sprintf(
+ "%s lists %s, and the build wrote no page served at that address, so a crawler following it is answered with the not-found page",
+ site.SitemapPath, l))
+ case want[l] && have[l] > 1 && !reported[l]:
+ reported[l] = true
+ details = append(details, fmt.Sprintf(
+ "%s lists %s %d times, and one page listed twice is one page fetched twice by everything that reads the file",
+ site.SitemapPath, l, have[l]))
+ }
+ }
+ return details
+}
+
+// Run builds the tree at root into a directory it throws away and compares the
+// sitemap it finds there against the pages beside it.
+func Run(root string, log io.Writer) error {
+ tmp, err := os.MkdirTemp("", "site-sitemap-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmp)
+
+ out := filepath.Join(tmp, site.OutputDir)
+ if _, err := site.Build(root, out, io.Discard); err != nil {
+ return fmt.Errorf("the build refused, so there is nothing to compare: %w", err)
+ }
+
+ produced, err := walk(out)
+ if err != nil {
+ return err
+ }
+ owed := site.SitemapAddresses(produced)
+
+ body, err := os.ReadFile(filepath.Join(out, filepath.FromSlash(site.SitemapPath)))
+ switch {
+ case os.IsNotExist(err) && len(owed) == 0:
+ fmt.Fprintf(log, "sitemap: the build produced no page and no %s, so this leg examined nothing\n", site.SitemapPath)
+ return nil
+ case os.IsNotExist(err):
+ fmt.Fprintf(log, "sitemap: 0 address(es) listed, against %d page(s) the build produced\n", len(owed))
+ fmt.Fprintf(log, " the build wrote %d page(s) and no %s, so nothing lists any of them\n", len(owed), site.SitemapPath)
+ return fmt.Errorf("sitemap: %d page(s) are listed nowhere", len(owed))
+ case err != nil:
+ return err
+ }
+
+ listed := Listed(body)
+ details := Decide(listed, owed)
+
+ fmt.Fprintf(log, "sitemap: %d address(es) listed, against %d page(s) the build produced\n", len(listed), len(owed))
+ if len(details) > 0 {
+ for _, d := range details {
+ fmt.Fprintf(log, " %s\n", d)
+ }
+ return fmt.Errorf("sitemap: %d disagreement(s) between the list and the output", len(details))
+ }
+ fmt.Fprintf(log, " every page the build wrote is listed once, and every entry has a page behind it\n")
+ return nil
+}
+
+// walk returns every file under out, named the way the host serves it. It is a
+// walk of the directory rather than of what the build said it wrote, which is
+// the whole point of this leg.
+func walk(out string) ([]string, error) {
+ var found []string
+ err := filepath.WalkDir(out, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ rel, err := filepath.Rel(out, p)
+ if err != nil {
+ return err
+ }
+ found = append(found, strings.TrimPrefix(filepath.ToSlash(rel), "./"))
+ return nil
+ })
+ return found, err
+}
diff --git a/internal/sitemap/sitemap_test.go b/internal/sitemap/sitemap_test.go
new file mode 100644
index 0000000..06e4ae2
--- /dev/null
+++ b/internal/sitemap/sitemap_test.go
@@ -0,0 +1,124 @@
+// The suite over the comparison between a sitemap and the output beside it.
+//
+// The two directions are driven through the decision rather than through a
+// build, because a build writes the file from the same rule the comparison
+// reads, so a case that went through one could never produce the disagreement
+// this leg exists to refuse. What the cases below hand it is the two lists.
+//
+// No case opens a window, binds a socket, reaches the network or needs anything
+// that is not in the toolchain.
+package sitemap
+
+import (
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/Flowfin/site/internal/site"
+)
+
+const (
+ root = site.Origin + "/"
+ privacy = site.Origin + "/privacy/"
+ legal = site.Origin + "/legal/"
+)
+
+// The near miss. A page lands in the output and the list does not carry it,
+// which is what a writer added after the sitemap is assembled produces. Every
+// other check stays green: the page is valid, it is served, and nothing tells a
+// crawler it is there.
+func TestAProducedPageThatIsListedNowhereIsRefused(t *testing.T) {
+ details := Decide([]string{root, privacy}, []string{root, privacy, legal})
+
+ if len(details) != 1 {
+ t.Fatalf("Decide gave %d detail(s), want the one missing page: %v", len(details), details)
+ }
+ if !strings.Contains(details[0], legal) {
+ t.Errorf("the failure does not name the page that is listed nowhere: %s", details[0])
+ }
+}
+
+// The same two lists with the entry restored. Without this the case above
+// proves that something reds rather than that this reds for its own reason.
+func TestTheSameListsAgreeingArePassed(t *testing.T) {
+ details := Decide([]string{root, privacy, legal}, []string{root, privacy, legal})
+
+ if len(details) != 0 {
+ t.Fatalf("Decide refused a list that matches the output: %v", details)
+ }
+}
+
+// The other direction. An address with no page behind it sends every client
+// that reads the file to the not-found page, and the site is the party claiming
+// the address exists.
+func TestAnEntryWithNoPageBehindItIsRefused(t *testing.T) {
+ details := Decide([]string{root, privacy, legal}, []string{root, privacy})
+
+ if len(details) != 1 {
+ t.Fatalf("Decide gave %d detail(s), want the one entry with nothing behind it: %v", len(details), details)
+ }
+ if !strings.Contains(details[0], legal) {
+ t.Errorf("the failure does not name the entry: %s", details[0])
+ }
+}
+
+// One page listed twice is one page fetched twice by everything that reads the
+// file, and both entries have a page behind them, so neither direction above
+// sees it.
+func TestAPageListedTwiceIsRefused(t *testing.T) {
+ details := Decide([]string{root, privacy, privacy}, []string{root, privacy})
+
+ if len(details) != 1 {
+ t.Fatalf("Decide gave %d detail(s), want the one duplicate: %v", len(details), details)
+ }
+ if !strings.Contains(details[0], privacy) {
+ t.Errorf("the failure does not name the duplicated entry: %s", details[0])
+ }
+}
+
+// Both directions at once are both reported. A run that stopped at the first
+// disagreement would cost a run per repair.
+func TestBothDirectionsAreReportedTogether(t *testing.T) {
+ details := Decide([]string{root, legal}, []string{root, privacy})
+
+ if len(details) != 2 {
+ t.Fatalf("Decide gave %d detail(s), want one per direction: %v", len(details), details)
+ }
+}
+
+// What the leg reads out of the file is the address element and nothing else.
+func TestListedReadsTheAddresses(t *testing.T) {
+ body := []byte(`
+
+ ` + root + `
+ ` + privacy + `
+
+`)
+
+ got := Listed(body)
+
+ if len(got) != 2 || got[0] != root || got[1] != privacy {
+ t.Fatalf("Listed gave %v, want the two addresses in the order the file states them", got)
+ }
+}
+
+// The tree this file sits in. It is the one case here that judges the real
+// output, and what it answers is whether this repository still agrees with
+// itself, which is a different question from whether the rule bites.
+func TestTheTreeAgreesWithItsOwnSitemap(t *testing.T) {
+ var log strings.Builder
+ if err := Run("../..", &log); err != nil {
+ t.Fatalf("%v\n%s", err, log.String())
+ }
+ if !strings.Contains(log.String(), "address(es) listed") {
+ t.Errorf("the run does not say what it compared:\n%s", log.String())
+ }
+}
+
+// A run says what it covered whether or not it found anything, so a leg that
+// compared an empty pair cannot be read as one that compared the site.
+func TestRunSaysWhatItCompared(t *testing.T) {
+ if err := Run("../..", io.Discard); err != nil {
+ t.Fatalf("Run over this tree: %v", err)
+ }
+}