feat(core): production upgrade with Makefile, expanded tracking tags, llms.txt stance, and UI copy report

.gitignore

@@ -1,9 +1,53 @@
+# Binaries and build artifacts
 /rem
 /rem.exe
-/rem-reports/
+*.exe
+*.dll
+*.so
+*.dylib
+*.test
 /dist/
+
+# Test artifacts and performance profiles
+coverage.out
+coverage.html
+*.pprof
+
+# Investigation output directories & local reports
+/rem-reports/
+/runs/
+.rem-home/
+*.log
+
+# Sensitive environments, credentials and tokens
 .env
 .env.*
 !.env.example
+*.pem
+*.key
+*.cert
+*.p12
+
+# OS files
 .DS_Store
-coverage.out
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# IDE and Editor directories
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.project
+.settings/
+
+# Temporary and transfer directories
+tmp/
+temp/
+.scratch/
+.rem-go-transfer.*/

Makefile

@@ -0,0 +1,50 @@
+# REM Go — production Makefile
+# Requires Go 1.26+.  Run `make` for the default (all) target.
+
+BINARY     := rem
+CMD        := ./cmd/rem
+GOPATH     ?= $(shell go env GOPATH)
+INSTALL_DIR := $(GOPATH)/bin
+DIST       := dist
+
+LDFLAGS    := -s -w
+BUILD_FLAGS := -trimpath -ldflags="$(LDFLAGS)"
+
+.PHONY: all build test smoke install cross lint clean
+
+## Default ─────────────────────────────────────────────────────────────────────
+all: build test lint
+
+## Build the binary for the current platform
+build:
+	go build $(BUILD_FLAGS) -o $(BINARY) $(CMD)
+
+## Run the full test suite with race detector
+test:
+	go test -count=1 -race ./...
+
+## Smoke-test the built binary against a live target.
+## Override the target with REM_LIVE_TARGET=<domain> (default: linear.app).
+smoke: build
+	REM_LIVE_TARGET=$${REM_LIVE_TARGET:-linear.app} bash smoke.sh ./$(BINARY)
+
+## Install the binary into $(GOPATH)/bin/
+install: build
+	install -m 0755 $(BINARY) $(INSTALL_DIR)/$(BINARY)
+	@echo "installed → $(INSTALL_DIR)/$(BINARY)"
+
+## Cross-compile release binaries into dist/
+cross:
+	mkdir -p $(DIST)
+	GOOS=linux  GOARCH=amd64 go build $(BUILD_FLAGS) -o $(DIST)/$(BINARY)-linux-amd64   $(CMD)
+	GOOS=darwin GOARCH=arm64 go build $(BUILD_FLAGS) -o $(DIST)/$(BINARY)-darwin-arm64  $(CMD)
+	@echo "cross-compiled binaries written to $(DIST)/"
+
+## Static analysis
+lint:
+	go vet ./...
+
+## Remove compiled artifacts
+clean:
+	rm -f $(BINARY)
+	rm -rf $(DIST)

README.md

@@ -22,13 +22,24 @@ rem demo
 From the source directory with Go 1.26 or later:
 
 ```sh
-go build -trimpath -o rem ./cmd/rem
+make build   # or: go build -trimpath -o rem ./cmd/rem
 ./rem doctor
 ./rem linear.app --only website,seo --no-ai --out ./rem-reports/linear
 ./rem validate ./rem-reports/linear/evidence.json ./rem-reports/linear/report.md
 ./rem web
 ```
 
+### Build & test automation
+
+```sh
+make build    # Build optimized binary with -trimpath and -ldflags="-s -w"
+make test     # Run full test suite with race detector (-race)
+make lint     # Run go vet ./...
+make smoke    # Run end-to-end smoke test on linear.app (or $REM_LIVE_TARGET)
+make install  # Install binary to $(GOPATH)/bin/rem
+make cross    # Cross-compile for linux/amd64 and darwin/arm64 into dist/
+```
+
 `./rem demo` opens a recorded Cal AI investigation with no requests to source providers. Use `--no-open --port 0` to print a dynamically allocated loopback URL without launching a browser. Choose **Evidence only** in the browser to collect and view reports without a model.
 
 To compare two snapshots:

internal/collect/seo.go

@@ -76,11 +76,29 @@ func SEO(ctx context.Context, get fetch.Getter, target model.Target) model.Resul
 	if llmsProblem != "" {
 		result.Unknowns = append(result.Unknowns, unknown("geo", "llms.txt: "+llmsProblem, llmsURL))
 	}
+	// llms.txt access policy: check whether the file allows or restricts LLM access.
+	var llmsAllows *bool
+	if llmsStatus == "present" && llmsText != "" {
+		lower := strings.ToLower(llmsText)
+		denies := strings.Contains(lower, "disallow") || strings.Contains(lower, "noai") ||
+			strings.Contains(lower, "no-ai") || strings.Contains(lower, "forbid") ||
+			strings.Contains(lower, "not permitted") || strings.Contains(lower, "prohibited")
+		grants := strings.Contains(lower, "allow") || strings.Contains(lower, "permitted") ||
+			strings.Contains(lower, "welcome") || strings.Contains(lower, "encouraged")
+		v := grants && !denies
+		llmsAllows = &v
+	}
 	if robotsStatus != "unavailable" || llmsStatus != "unavailable" {
 		var stance []string
 		switch llmsStatus {
 		case "present":
-			stance = append(stance, "has llms.txt")
+			if llmsAllows != nil && !*llmsAllows {
+				stance = append(stance, "has llms.txt (restricts LLMs)")
+			} else if llmsAllows != nil && *llmsAllows {
+				stance = append(stance, "has llms.txt (allows LLMs)")
+			} else {
+				stance = append(stance, "has llms.txt")
+			}
 		case "absent":
 			stance = append(stance, "llms.txt absent")
 		default:
@@ -100,7 +118,7 @@ func SEO(ctx context.Context, get fetch.Getter, target model.Target) model.Resul
 			sourceURL = llmsURL
 		}
 		result.Evidence = append(result.Evidence, evidence("seo", "observation", "AI-search stance: "+strings.Join(stance, ", "), sourceURL, map[string]any{
-			"robots": robots, "robotsStatus": robotsStatus, "robotsURL": robotsURL, "llmsTxt": llmsText, "llmsStatus": llmsStatus, "llmsURL": llmsURL,
+			"robots": robots, "robotsStatus": robotsStatus, "robotsURL": robotsURL, "llmsTxt": llmsText, "llmsStatus": llmsStatus, "llmsURL": llmsURL, "llmsAllowsLLMs": llmsAllows,
 		}))
 	}
 	collectSitemaps(ctx, get, base, robots, &result)

internal/collect/tracking.go

@@ -89,6 +89,10 @@ var signatures = []signature{
 	tag("loops", "Loops", "email", "lifecycle", `(?i)<host>app\.loops\.so<end>`, ""),
 	tag("braze", "Braze", "email", "lifecycle", `(?i)<host>js\.appboycdn\.com<end>|<host>sdk\.iad-\d+\.braze\.com<end>`, ""),
 	tag("iterable", "Iterable", "email", "lifecycle", `(?i)<host>iterable\.com/api<end>|<host>js\.iterable\.com<end>`, ""),
+	tag("amazon_ads", "Amazon Ads", "ads_pixel", "paid_amazon", `(?i)<host>s\.amazon-adsystem\.com<end>|amazon_advertising_tag`, ""),
+	tag("criteo", "Criteo", "ads_pixel", "paid_programmatic", `(?i)<host>static\.criteo\.net/js/ld/publishertag<end>`, ""),
+	tag("outbrain", "Outbrain", "ads_pixel", "paid_programmatic", `(?i)<host>widgets\.outbrain\.com<end>`, ""),
+	tag("taboola", "Taboola", "ads_pixel", "paid_programmatic", `(?i)<host>cdn\.taboola\.com<end>`, ""),
 	tag("intercom", "Intercom", "chat", "", `(?i)<host>widget\.intercom\.io<end>|<host>js\.intercomcdn\.com<end>`, ""),
 	tag("crisp", "Crisp", "chat", "", `(?i)<host>client\.crisp\.chat<end>`, ""),
 	tag("drift", "Drift", "chat", "sales_led", `(?i)<host>js\.driftt\.com<end>`, ""),

internal/web/app.js

@@ -461,6 +461,21 @@ function displayRun(id, data) {
   const path = `/api/runs/${encodeURIComponent(id)}`;
   $("markdown-link").href = `${path}/report.md`;
   $("markdown-link").hidden = !data.report;
+  const copyBtn = $("copy-report-btn");
+  if (copyBtn) {
+    copyBtn.hidden = !data.report;
+    copyBtn.onclick = async () => {
+      try {
+        await navigator.clipboard.writeText(data.report);
+        const originalText = copyBtn.textContent;
+        copyBtn.textContent = "Copied!";
+        setTimeout(() => { copyBtn.textContent = originalText; }, 2000);
+      } catch (e) {
+        copyBtn.textContent = "Copy failed";
+        setTimeout(() => { copyBtn.textContent = "Copy report"; }, 2000);
+      }
+    };
+  }
   $("evidence-link").href = `${path}/evidence.json`;
   $("metrics").replaceChildren(...[
     [current.evidence.length, "Evidence items"], [current.content.length, "Ads & posts"], [bundle.unknowns?.length || 0, "Unknowns"],

internal/web/index.html

@@ -68,7 +68,7 @@ <h2 id="progress-title">Investigation in progress</h2>
       </div>
       <div class="run-title">
         <div><h1 id="brand"></h1><p id="target-meta"></p></div>
-        <div class="download-links"><a class="secondary" id="markdown-link">Report .md</a><a class="secondary" id="evidence-link">Evidence .json</a></div>
+        <div class="download-links"><button class="secondary" id="copy-report-btn" type="button" hidden>Copy report</button><a class="secondary" id="markdown-link">Report .md</a><a class="secondary" id="evidence-link">Evidence .json</a></div>
       </div>
       <div class="metrics" id="metrics"></div>
       <p class="notice" id="report-warning" hidden></p>