diff --git a/CHANGELOG.md b/CHANGELOG.md
index 242f120..7a58cbb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,3 +64,10 @@ there is no earlier bundle to compare it against.
it, packed so that two runs of one source produce one archive.
- The bill of materials states the version of the thing it is about, so two
archives are distinguishable by the document each one carries.
+- The build writes `robots.txt` and `sitemap.xml`. Both are asked for by clients
+ without any page linking them, and a bundle without them answers those
+ requests with the not-found page. The sitemap lists every page the build wrote
+ except the not-found one, at the address each is served at, and carries no
+ date, so two builds of one source still produce one set of bytes.
+- The gate refuses a bundle whose sitemap disagrees with the pages beside it, in
+ both directions: a page listed nowhere, and an address with no page behind it.
diff --git a/internal/gate/gate.go b/internal/gate/gate.go
index c2ef990..6db1266 100644
--- a/internal/gate/gate.go
+++ b/internal/gate/gate.go
@@ -21,6 +21,7 @@ import (
"github.com/Flowfin/site/internal/invariant"
"github.com/Flowfin/site/internal/link"
"github.com/Flowfin/site/internal/site"
+ "github.com/Flowfin/site/internal/sitemap"
)
// A leg is one thing the gate decides. Its report is the sentence printed
@@ -38,6 +39,7 @@ func legs() []leg {
{"test", testLeg},
{"build", buildLeg},
{"links", linksLeg},
+ {"sitemap", sitemapLeg},
{"invariants", invariantsLeg},
}
}
@@ -209,6 +211,24 @@ func linksLeg(root string) (string, error) {
return strings.TrimSpace(lines[len(lines)-1]), nil
}
+// sitemapLeg refuses a sitemap that disagrees with the pages beside it. It sits
+// after the links leg because both walk what the build produced and the links
+// leg answers the more basic question: a site whose pages point at files nobody
+// wrote is broken for a reader, where a sitemap that has drifted is broken only
+// for a crawler.
+func sitemapLeg(root string) (string, error) {
+ var log strings.Builder
+ if err := sitemap.Run(root, &log); err != nil {
+ lines := strings.Split(strings.TrimRight(log.String(), "\n"), "\n")
+ return "", fmt.Errorf("%v:\n%s", err, indent(lines))
+ }
+ // The last line the comparison wrote is the one that says what it
+ // covered, in its own words, so a run that examined nothing says so here
+ // rather than reporting a count assembled in this file.
+ lines := strings.Split(strings.TrimRight(log.String(), "\n"), "\n")
+ return strings.TrimSpace(lines[len(lines)-1]), nil
+}
+
// invariantsLeg decides the rules that can be read off the tree and off the
// output a build produces. The rows live in their own package rather than here,
// so the same set is decided by this leg and by the workflow that reports it
diff --git a/internal/reproduce/reproduce_test.go b/internal/reproduce/reproduce_test.go
index 447b56c..7b76d92 100644
--- a/internal/reproduce/reproduce_test.go
+++ b/internal/reproduce/reproduce_test.go
@@ -54,12 +54,18 @@ func write(t *testing.T, root, name, body string) {
// The neighbour, and it is the real build rather than a stand-in: a tree whose
// build reads only what is committed produces the same bytes twice, and the run
// says how many files it compared rather than passing silently.
+//
+// The count is the page and the two files a crawler asks for. It is written out
+// rather than derived, because those two are the ones a build could most easily
+// make unreproducible: a sitemap may state when each page last changed, and one
+// carrying today's date would pass every other check in this tree and red only
+// here.
func TestTwoBuildsOfTheRealGeneratorAgree(t *testing.T) {
var log bytes.Buffer
if err := Run(tree(t), &log); err != nil {
t.Fatalf("Run refused two builds of one source: %v\n%s", err, log.String())
}
- if !strings.Contains(log.String(), "1 file(s), identical in both builds") {
+ if !strings.Contains(log.String(), "3 file(s), identical in both builds") {
t.Errorf("the run did not say what it compared; it said:\n%s", log.String())
}
}
diff --git a/internal/site/crawler.go b/internal/site/crawler.go
new file mode 100644
index 0000000..a0d4863
--- /dev/null
+++ b/internal/site/crawler.go
@@ -0,0 +1,146 @@
+// The files a crawler asks for without any page linking them, and what the
+// build writes into them.
+//
+// A crawler asks for the exclusion file before it asks for anything else, and it
+// asks again on every site it has not seen answer. Where the repository provides
+// none, the host answers with the not-found page: a request the reader's host
+// serves for nothing, and an answer that carries none of what was asked for.
+// Producing the file is what turns that into one short answer.
+//
+// The sitemap is generated rather than written down. Twelve of the addresses
+// this site will have come from a file somebody else edits, so a hand-written
+// list is wrong the day a row is added and nothing about the wrong list looks
+// wrong. What the generated one lists is what this build just wrote.
+//
+// Neither file carries a date. A sitemap may state when each page last changed,
+// and a build that wrote today's date into one would produce different bytes
+// from the same source on two days, against a check that exists to compare
+// exactly that. What it costs is that a crawler is told which addresses exist
+// and not which of them moved, and the second half is worth less here than a
+// build somebody can reproduce.
+package site
+
+import (
+ "encoding/xml"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strings"
+)
+
+// RobotsPath and SitemapPath are where the two files land. Neither name is this
+// repository's choice: the first is the one address a crawler asks for before it
+// has read anything, and the second is found only because the first names it.
+const (
+ RobotsPath = "robots.txt"
+ SitemapPath = "sitemap.xml"
+)
+
+// SitemapAddresses is which of the produced files a sitemap lists, and at which
+// address. It takes paths relative to the output directory.
+//
+// It is exported because two readers need the same answer: the writer below,
+// which turns it into the file, and the leg that walks the output afterwards and
+// compares the file against it. Stating the rule in both places would state it
+// twice, and the second statement would agree with the first on the day it was
+// written and never be checked against it again.
+//
+// The not-found page is the one page left out, and it is left out rather than
+// forgotten. A sitemap is a list of addresses a crawler is invited to fetch and
+// index, and that page is served in answer to addresses that are not its own, so
+// listing it asks for an index entry that sends a reader to an error page under
+// an address the site says it has. Every other produced page is listed, and
+// anything that is not a page is not, because what a sitemap carries is the
+// addresses a reader can be sent to.
+//
+// The addresses are sorted, so the file the build writes is a property of which
+// pages exist rather than of the order the writers happen to run in.
+func SitemapAddresses(produced []string) []string {
+ var addresses []string
+ for _, name := range produced {
+ name = strings.TrimPrefix(path.Clean(filepath.ToSlash(name)), "/")
+ if !strings.HasSuffix(name, ".html") || name == NotFoundPath {
+ continue
+ }
+ addresses = append(addresses, Origin+addressOf(name))
+ }
+ sort.Strings(addresses)
+ return addresses
+}
+
+// writeSitemap writes the list of addresses this build produced, and reports
+// what it wrote.
+//
+// A build that produced no page writes no sitemap and says so, rather than
+// writing a list of nothing. An empty list is a file that says this site has no
+// pages, which is a statement about the site rather than a statement that the
+// build had nothing to say.
+func writeSitemap(out, label string, written []string, log io.Writer) ([]string, error) {
+ addresses := SitemapAddresses(relativeTo(label, written))
+ if len(addresses) == 0 {
+ fmt.Fprintf(log, "the build produced no page, so no %s was written\n", SitemapPath)
+ return nil, nil
+ }
+
+ var body strings.Builder
+ body.WriteString(xml.Header)
+ body.WriteString(`` + "\n")
+ for _, a := range addresses {
+ var escaped strings.Builder
+ if err := xml.EscapeText(&escaped, []byte(a)); err != nil {
+ return nil, fmt.Errorf("writing %s: %w", SitemapPath, err)
+ }
+ fmt.Fprintf(&body, " %s\n", escaped.String())
+ }
+ body.WriteString("\n")
+
+ name := filepath.Join(out, filepath.FromSlash(SitemapPath))
+ if err := os.WriteFile(name, []byte(body.String()), 0o644); err != nil {
+ return nil, fmt.Errorf("writing %s: %w", SitemapPath, err)
+ }
+ slashed := path.Join(label, SitemapPath)
+ fmt.Fprintf(log, "wrote %s (%d bytes, %d address(es) listed)\n", slashed, body.Len(), len(addresses))
+ return []string{slashed}, nil
+}
+
+// writeRobots writes the exclusion file. Nothing on this site is kept out of an
+// index, so the file says that plainly and spends the rest of itself naming
+// where the list of addresses is.
+//
+// It names the sitemap only where one was written. A robots file pointing at an
+// address the build did not produce sends the one client that reads it to the
+// not-found page, which is the failure this pair exists to remove, arriving from
+// the file that was supposed to remove it.
+func writeRobots(out, label string, sitemap []string, log io.Writer) ([]string, error) {
+ var body strings.Builder
+ body.WriteString("# Nothing on this site is kept out of an index. This file exists so that a\n")
+ body.WriteString("# crawler asking for it is answered rather than served the not-found page.\n")
+ body.WriteString("User-agent: *\n")
+ body.WriteString("Disallow:\n")
+ if len(sitemap) > 0 {
+ fmt.Fprintf(&body, "\nSitemap: %s/%s\n", Origin, SitemapPath)
+ }
+
+ name := filepath.Join(out, filepath.FromSlash(RobotsPath))
+ if err := os.WriteFile(name, []byte(body.String()), 0o644); err != nil {
+ return nil, fmt.Errorf("writing %s: %w", RobotsPath, err)
+ }
+ slashed := path.Join(label, RobotsPath)
+ fmt.Fprintf(log, "wrote %s (%d bytes, %d sitemap(s) named)\n", slashed, body.Len(), len(sitemap))
+ return []string{slashed}, nil
+}
+
+// relativeTo strips the output label off the paths the writers report, so what
+// this file works in is the addresses the host will serve rather than wherever
+// the run happened to render.
+func relativeTo(label string, written []string) []string {
+ prefix := filepath.ToSlash(label) + "/"
+ out := make([]string, 0, len(written))
+ for _, w := range written {
+ out = append(out, strings.TrimPrefix(filepath.ToSlash(w), prefix))
+ }
+ return out
+}
diff --git a/internal/site/crawler_test.go b/internal/site/crawler_test.go
new file mode 100644
index 0000000..d28a0ed
--- /dev/null
+++ b/internal/site/crawler_test.go
@@ -0,0 +1,142 @@
+// The suite over the two files a crawler asks for.
+//
+// The cases about which addresses belong in a sitemap drive the rule directly
+// rather than through a build, because the rule is what the leg over the output
+// reads and a case that went through a build would be testing the build.
+package site
+
+import (
+ "io"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// The not-found page is produced, is a page, and is the one page a sitemap must
+// not carry. Listing it asks a crawler to index the document a host serves for
+// addresses this site does not have, under an address the site says it has.
+func TestSitemapLeavesOutTheNotFoundPage(t *testing.T) {
+ produced := []string{"index.html", "privacy/index.html", NotFoundPath}
+
+ got := SitemapAddresses(produced)
+
+ for _, a := range got {
+ if strings.HasSuffix(a, "/"+NotFoundPath) {
+ t.Fatalf("the sitemap lists the not-found page: %v", got)
+ }
+ }
+ if len(got) != 2 {
+ t.Fatalf("SitemapAddresses(%v) = %v, want the two pages that are not the not-found one", produced, got)
+ }
+}
+
+// A directory address is served by the index document inside it, and the
+// sitemap states the address rather than the file. A crawler told to fetch the
+// file would index a second address for a page that already has one.
+func TestSitemapStatesTheAddressAndNotTheFile(t *testing.T) {
+ got := SitemapAddresses([]string{"index.html", "privacy/index.html"})
+
+ want := []string{Origin + "/", Origin + "/privacy/"}
+ if len(got) != len(want) {
+ t.Fatalf("SitemapAddresses gave %v, want %v", got, want)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Errorf("entry %d is %s, want %s", i, got[i], want[i])
+ }
+ }
+}
+
+// What a sitemap carries is addresses a reader can be sent to. The reporting
+// route, the exclusion file and the sitemap itself are all produced and none of
+// them is a page.
+func TestSitemapCarriesOnlyPages(t *testing.T) {
+ produced := []string{
+ "index.html",
+ ".well-known/security.txt",
+ RobotsPath,
+ SitemapPath,
+ "nested/style.css",
+ }
+
+ got := SitemapAddresses(produced)
+
+ if len(got) != 1 || got[0] != Origin+"/" {
+ t.Fatalf("SitemapAddresses(%v) = %v, want the one page", produced, got)
+ }
+}
+
+// The order of the file is a property of which pages exist. Two runs over the
+// same tree produce the same bytes, which is what the check that builds twice
+// compares, and the order the writers happen to run in is not part of it.
+func TestSitemapIsSortedWhateverOrderThePagesArrivedIn(t *testing.T) {
+ first := SitemapAddresses([]string{"privacy/index.html", "index.html", "legal/index.html"})
+ second := SitemapAddresses([]string{"index.html", "legal/index.html", "privacy/index.html"})
+
+ if len(first) != len(second) {
+ t.Fatalf("the two orders gave %v and %v", first, second)
+ }
+ for i := range first {
+ if first[i] != second[i] {
+ t.Fatalf("the two orders gave %v and %v", first, second)
+ }
+ }
+ if first[0] != Origin+"/" {
+ t.Errorf("the list opens with %s, and sorted it opens with the site root", first[0])
+ }
+}
+
+// A build that produced no page writes no sitemap and says so. An empty list is
+// a file stating that this site has no pages, which is a claim rather than a
+// report that there was nothing to write.
+func TestNoPageMeansNoSitemapAndASaidReason(t *testing.T) {
+ out := t.TempDir()
+
+ var log strings.Builder
+ written, err := writeSitemap(out, "dist", []string{"dist/" + RobotsPath}, &log)
+ if err != nil {
+ t.Fatalf("writeSitemap: %v", err)
+ }
+ if len(written) != 0 {
+ t.Errorf("writeSitemap reported %v out of a build with no page", written)
+ }
+ if !strings.Contains(log.String(), "produced no page") {
+ t.Errorf("the run does not say why nothing was written:\n%s", log.String())
+ }
+}
+
+// The exclusion file names the sitemap only where one was written. A robots file
+// pointing at an address the build did not produce sends the one client that
+// reads it to the not-found page, which is the failure this pair exists to
+// remove.
+func TestRobotsNamesNoSitemapThatWasNotWritten(t *testing.T) {
+ out := t.TempDir()
+
+ if _, err := writeRobots(out, "dist", nil, io.Discard); err != nil {
+ t.Fatalf("writeRobots: %v", err)
+ }
+ got := read(t, filepath.Join(out, RobotsPath))
+
+ if strings.Contains(got, "Sitemap:") {
+ t.Errorf("the file names a sitemap the build did not write:\n%s", got)
+ }
+ if !strings.Contains(got, "User-agent: *") {
+ t.Errorf("the file excludes nobody and does not say so:\n%s", got)
+ }
+}
+
+// With a sitemap beside it the file names it, at the address a crawler will ask
+// for rather than at the path the build wrote.
+func TestRobotsNamesTheSitemapThatWasWritten(t *testing.T) {
+ out := t.TempDir()
+
+ if _, err := writeRobots(out, "dist", []string{"dist/" + SitemapPath}, io.Discard); err != nil {
+ t.Fatalf("writeRobots: %v", err)
+ }
+ got := read(t, filepath.Join(out, RobotsPath))
+
+ want := "Sitemap: " + Origin + "/" + SitemapPath
+ if !strings.Contains(got, want) {
+ t.Errorf("the file does not carry %q; it is:\n%s", want, got)
+ }
+}
diff --git a/internal/site/privacy_test.go b/internal/site/privacy_test.go
index a641fd5..9d3b871 100644
--- a/internal/site/privacy_test.go
+++ b/internal/site/privacy_test.go
@@ -277,7 +277,7 @@ func TestBuildWritesThePrivacyPageAndSaysWhatIsOnIt(t *testing.T) {
if err != nil {
t.Fatalf("the build refused: %v\n%s", err, log.String())
}
- if len(written) != 2 || written[1] != "dist/privacy/index.html" {
+ if !wrote(written, "dist/privacy/index.html") {
t.Fatalf("the build wrote %q", written)
}
if !strings.Contains(log.String(), "1 checked, 0 promised, 1 residual") {
@@ -327,8 +327,8 @@ func TestBuildSaysWhenThereIsNoPrivacyProse(t *testing.T) {
if err != nil {
t.Fatalf("the build refused: %v\n%s", err, log.String())
}
- if len(written) != 1 {
- t.Fatalf("the build wrote %q", written)
+ if wrote(written, "dist/privacy/index.html") {
+ t.Fatalf("the build wrote a privacy page out of a tree that carries no prose for one: %q", written)
}
if !strings.Contains(log.String(), "no content/privacy.txt in the tree") {
t.Errorf("the run passed over the absence:\n%s", log.String())
diff --git a/internal/site/site.go b/internal/site/site.go
index ba58be8..4473d64 100644
--- a/internal/site/site.go
+++ b/internal/site/site.go
@@ -186,6 +186,24 @@ func Build(root, outDir string, log io.Writer) ([]string, error) {
}
written = append(written, copied...)
+ // The two files nothing links are written last, after everything that can
+ // put a page into the output, because the sitemap is a list of what is
+ // above it and anything landing underneath it would be served and listed
+ // nowhere. That the ordering holds is not left to this comment: the leg
+ // over the output walks the directory afterwards and compares what the
+ // file lists against what is beside it.
+ sitemap, err := writeSitemap(out, label, written, log)
+ if err != nil {
+ return nil, err
+ }
+ written = append(written, sitemap...)
+
+ robots, err := writeRobots(out, label, sitemap, log)
+ if err != nil {
+ return nil, err
+ }
+ written = append(written, robots...)
+
fmt.Fprintf(log, "%d file(s) written into %s\n", len(written), label)
return written, nil
}
diff --git a/internal/site/site_test.go b/internal/site/site_test.go
index 12e5381..bad1d7a 100644
--- a/internal/site/site_test.go
+++ b/internal/site/site_test.go
@@ -40,6 +40,19 @@ func tree(t *testing.T, prose string) string {
return root
}
+// wrote answers whether the build reported a path. A case that asked instead
+// how many paths came back would be a case about the whole set of files the
+// build writes, which is a thing every new writer moves and no case here is
+// about.
+func wrote(written []string, want string) bool {
+ for _, w := range written {
+ if w == want {
+ return true
+ }
+ }
+ return false
+}
+
func mkdir(t *testing.T, dir string) {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
@@ -74,8 +87,8 @@ func TestBuildJoinsAWrappedParagraphIntoOneSentence(t *testing.T) {
if err != nil {
t.Fatalf("Build: %v", err)
}
- if len(written) != 1 || written[0] != "dist/index.html" {
- t.Fatalf("Build reported %v, want [dist/index.html]", written)
+ if !wrote(written, "dist/index.html") {
+ t.Fatalf("Build reported %v, and none of it is dist/index.html", written)
}
got := read(t, filepath.Join(root, OutputDir, "index.html"))
@@ -146,8 +159,10 @@ func TestBuildWritesAnAbsoluteOutputDirectoryWhereItWasAsked(t *testing.T) {
if _, err := os.Stat(filepath.Join(root, OutputDir)); !os.IsNotExist(err) {
t.Errorf("an absolute output directory still produced %s in the tree", filepath.Join(root, OutputDir))
}
- if len(written) != 1 {
- t.Errorf("Build reported %v, want one file", written)
+ for _, w := range written {
+ if !strings.HasPrefix(w, filepath.ToSlash(elsewhere)+"/") {
+ t.Errorf("Build reported %s, which is not under the directory it was asked for", w)
+ }
}
}
@@ -207,8 +222,8 @@ func TestBuildCopiesAssetsByteForByte(t *testing.T) {
if got := read(t, copied); got != body {
t.Errorf("the asset came out as %q, want %q", got, body)
}
- if len(written) != 2 || written[1] != "dist/nested/style.css" {
- t.Errorf("Build reported %v, want the page and dist/nested/style.css", written)
+ if !wrote(written, "dist/nested/style.css") {
+ t.Errorf("Build reported %v, and none of it is the copied asset", written)
}
}
diff --git a/internal/sitemap/sitemap.go b/internal/sitemap/sitemap.go
new file mode 100644
index 0000000..d44def1
--- /dev/null
+++ b/internal/sitemap/sitemap.go
@@ -0,0 +1,158 @@
+// Package sitemap refuses a sitemap that disagrees with the pages the build
+// wrote.
+//
+// The file is generated, so on the day it is written the two agree by
+// construction and a check over them proves nothing. What this refuses is the
+// day after. A page written by something that runs after the list is assembled,
+// or beside the flow that assembles it, lands in the output and is named
+// nowhere: the build is green, the page is served, and the only party who finds
+// out is a crawler that is never told to ask for it.
+//
+// The other direction is the one that is worse to serve. An address listed with
+// no page behind it sends every client that reads the list to the not-found
+// page, repeatedly, and the site is the party claiming the address exists.
+//
+// It walks the output rather than reading the list of paths the build reported,
+// so what it compares is the file on disk against the files on disk. A walk that
+// trusted the writer's own account of what it wrote would agree with the writer
+// about a page the writer never mentioned.
+package sitemap
+
+import (
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/Flowfin/site/internal/site"
+)
+
+// location is one address a sitemap states. The file is generated by this
+// repository and read by clients that accept far less than a parser would, so
+// this reads the one element it is about rather than the document around it: a
+// sitemap this build did not write is not a case that arises, and a whole XML
+// reader here would be apparatus nobody maintains.
+var location = regexp.MustCompile(`(?is)\s*([^<]*?)\s*`)
+
+// Listed returns the addresses a sitemap states, in the order it states them.
+func Listed(body []byte) []string {
+ var out []string
+ for _, m := range location.FindAllSubmatch(body, -1) {
+ out = append(out, string(m[1]))
+ }
+ return out
+}
+
+// Decide returns one detail per disagreement between what a sitemap lists and
+// what the produced pages are owed, in the order each side was given.
+//
+// Three things are refused and each names what it costs a reader, because the
+// two directions fail for opposite reasons and a failure saying only that the
+// two lists differ leaves the next person to work out which way round it is.
+func Decide(listed, owed []string) []string {
+ have := map[string]int{}
+ for _, l := range listed {
+ have[l]++
+ }
+ want := map[string]bool{}
+ for _, o := range owed {
+ want[o] = true
+ }
+
+ var details []string
+ for _, o := range owed {
+ if have[o] == 0 {
+ details = append(details, fmt.Sprintf(
+ "the build produced a page served at %s and %s lists no entry for it, so nothing ever tells a crawler the page is there",
+ o, site.SitemapPath))
+ }
+ }
+ reported := map[string]bool{}
+ for _, l := range listed {
+ switch {
+ case !want[l] && !reported[l]:
+ reported[l] = true
+ details = append(details, fmt.Sprintf(
+ "%s lists %s, and the build wrote no page served at that address, so a crawler following it is answered with the not-found page",
+ site.SitemapPath, l))
+ case want[l] && have[l] > 1 && !reported[l]:
+ reported[l] = true
+ details = append(details, fmt.Sprintf(
+ "%s lists %s %d times, and one page listed twice is one page fetched twice by everything that reads the file",
+ site.SitemapPath, l, have[l]))
+ }
+ }
+ return details
+}
+
+// Run builds the tree at root into a directory it throws away and compares the
+// sitemap it finds there against the pages beside it.
+func Run(root string, log io.Writer) error {
+ tmp, err := os.MkdirTemp("", "site-sitemap-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmp)
+
+ out := filepath.Join(tmp, site.OutputDir)
+ if _, err := site.Build(root, out, io.Discard); err != nil {
+ return fmt.Errorf("the build refused, so there is nothing to compare: %w", err)
+ }
+
+ produced, err := walk(out)
+ if err != nil {
+ return err
+ }
+ owed := site.SitemapAddresses(produced)
+
+ body, err := os.ReadFile(filepath.Join(out, filepath.FromSlash(site.SitemapPath)))
+ switch {
+ case os.IsNotExist(err) && len(owed) == 0:
+ fmt.Fprintf(log, "sitemap: the build produced no page and no %s, so this leg examined nothing\n", site.SitemapPath)
+ return nil
+ case os.IsNotExist(err):
+ fmt.Fprintf(log, "sitemap: 0 address(es) listed, against %d page(s) the build produced\n", len(owed))
+ fmt.Fprintf(log, " the build wrote %d page(s) and no %s, so nothing lists any of them\n", len(owed), site.SitemapPath)
+ return fmt.Errorf("sitemap: %d page(s) are listed nowhere", len(owed))
+ case err != nil:
+ return err
+ }
+
+ listed := Listed(body)
+ details := Decide(listed, owed)
+
+ fmt.Fprintf(log, "sitemap: %d address(es) listed, against %d page(s) the build produced\n", len(listed), len(owed))
+ if len(details) > 0 {
+ for _, d := range details {
+ fmt.Fprintf(log, " %s\n", d)
+ }
+ return fmt.Errorf("sitemap: %d disagreement(s) between the list and the output", len(details))
+ }
+ fmt.Fprintf(log, " every page the build wrote is listed once, and every entry has a page behind it\n")
+ return nil
+}
+
+// walk returns every file under out, named the way the host serves it. It is a
+// walk of the directory rather than of what the build said it wrote, which is
+// the whole point of this leg.
+func walk(out string) ([]string, error) {
+ var found []string
+ err := filepath.WalkDir(out, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ rel, err := filepath.Rel(out, p)
+ if err != nil {
+ return err
+ }
+ found = append(found, strings.TrimPrefix(filepath.ToSlash(rel), "./"))
+ return nil
+ })
+ return found, err
+}
diff --git a/internal/sitemap/sitemap_test.go b/internal/sitemap/sitemap_test.go
new file mode 100644
index 0000000..06e4ae2
--- /dev/null
+++ b/internal/sitemap/sitemap_test.go
@@ -0,0 +1,124 @@
+// The suite over the comparison between a sitemap and the output beside it.
+//
+// The two directions are driven through the decision rather than through a
+// build, because a build writes the file from the same rule the comparison
+// reads, so a case that went through one could never produce the disagreement
+// this leg exists to refuse. What the cases below hand it is the two lists.
+//
+// No case opens a window, binds a socket, reaches the network or needs anything
+// that is not in the toolchain.
+package sitemap
+
+import (
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/Flowfin/site/internal/site"
+)
+
+const (
+ root = site.Origin + "/"
+ privacy = site.Origin + "/privacy/"
+ legal = site.Origin + "/legal/"
+)
+
+// The near miss. A page lands in the output and the list does not carry it,
+// which is what a writer added after the sitemap is assembled produces. Every
+// other check stays green: the page is valid, it is served, and nothing tells a
+// crawler it is there.
+func TestAProducedPageThatIsListedNowhereIsRefused(t *testing.T) {
+ details := Decide([]string{root, privacy}, []string{root, privacy, legal})
+
+ if len(details) != 1 {
+ t.Fatalf("Decide gave %d detail(s), want the one missing page: %v", len(details), details)
+ }
+ if !strings.Contains(details[0], legal) {
+ t.Errorf("the failure does not name the page that is listed nowhere: %s", details[0])
+ }
+}
+
+// The same two lists with the entry restored. Without this the case above
+// proves that something reds rather than that this reds for its own reason.
+func TestTheSameListsAgreeingArePassed(t *testing.T) {
+ details := Decide([]string{root, privacy, legal}, []string{root, privacy, legal})
+
+ if len(details) != 0 {
+ t.Fatalf("Decide refused a list that matches the output: %v", details)
+ }
+}
+
+// The other direction. An address with no page behind it sends every client
+// that reads the file to the not-found page, and the site is the party claiming
+// the address exists.
+func TestAnEntryWithNoPageBehindItIsRefused(t *testing.T) {
+ details := Decide([]string{root, privacy, legal}, []string{root, privacy})
+
+ if len(details) != 1 {
+ t.Fatalf("Decide gave %d detail(s), want the one entry with nothing behind it: %v", len(details), details)
+ }
+ if !strings.Contains(details[0], legal) {
+ t.Errorf("the failure does not name the entry: %s", details[0])
+ }
+}
+
+// One page listed twice is one page fetched twice by everything that reads the
+// file, and both entries have a page behind them, so neither direction above
+// sees it.
+func TestAPageListedTwiceIsRefused(t *testing.T) {
+ details := Decide([]string{root, privacy, privacy}, []string{root, privacy})
+
+ if len(details) != 1 {
+ t.Fatalf("Decide gave %d detail(s), want the one duplicate: %v", len(details), details)
+ }
+ if !strings.Contains(details[0], privacy) {
+ t.Errorf("the failure does not name the duplicated entry: %s", details[0])
+ }
+}
+
+// Both directions at once are both reported. A run that stopped at the first
+// disagreement would cost a run per repair.
+func TestBothDirectionsAreReportedTogether(t *testing.T) {
+ details := Decide([]string{root, legal}, []string{root, privacy})
+
+ if len(details) != 2 {
+ t.Fatalf("Decide gave %d detail(s), want one per direction: %v", len(details), details)
+ }
+}
+
+// What the leg reads out of the file is the address element and nothing else.
+func TestListedReadsTheAddresses(t *testing.T) {
+ body := []byte(`
+
+ ` + root + `
+ ` + privacy + `
+
+`)
+
+ got := Listed(body)
+
+ if len(got) != 2 || got[0] != root || got[1] != privacy {
+ t.Fatalf("Listed gave %v, want the two addresses in the order the file states them", got)
+ }
+}
+
+// The tree this file sits in. It is the one case here that judges the real
+// output, and what it answers is whether this repository still agrees with
+// itself, which is a different question from whether the rule bites.
+func TestTheTreeAgreesWithItsOwnSitemap(t *testing.T) {
+ var log strings.Builder
+ if err := Run("../..", &log); err != nil {
+ t.Fatalf("%v\n%s", err, log.String())
+ }
+ if !strings.Contains(log.String(), "address(es) listed") {
+ t.Errorf("the run does not say what it compared:\n%s", log.String())
+ }
+}
+
+// A run says what it covered whether or not it found anything, so a leg that
+// compared an empty pair cannot be read as one that compared the site.
+func TestRunSaysWhatItCompared(t *testing.T) {
+ if err := Run("../..", io.Discard); err != nil {
+ t.Fatalf("Run over this tree: %v", err)
+ }
+}