diff --git a/.gitignore b/.gitignore index 3490689..5fde93f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,53 @@ +# Binaries and build artifacts /rem /rem.exe -/rem-reports/ +*.exe +*.dll +*.so +*.dylib +*.test /dist/ + +# Test artifacts and performance profiles +coverage.out +coverage.html +*.pprof + +# Investigation output directories & local reports +/rem-reports/ +/runs/ +.rem-home/ +*.log + +# Sensitive environments, credentials and tokens .env .env.* !.env.example +*.pem +*.key +*.cert +*.p12 + +# OS files .DS_Store -coverage.out +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# IDE and Editor directories +.idea/ +.vscode/ +*.swp +*.swo +*~ +.project +.settings/ + +# Temporary and transfer directories +tmp/ +temp/ +.scratch/ +.rem-go-transfer.*/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3638b31 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +# REM Go — production Makefile +# Requires Go 1.26+. Run `make` for the default (all) target. + +BINARY := rem +CMD := ./cmd/rem +GOPATH ?= $(shell go env GOPATH) +INSTALL_DIR := $(GOPATH)/bin +DIST := dist + +LDFLAGS := -s -w +BUILD_FLAGS := -trimpath -ldflags="$(LDFLAGS)" + +.PHONY: all build test smoke install cross lint clean + +## Default ───────────────────────────────────────────────────────────────────── +all: build test lint + +## Build the binary for the current platform +build: + go build $(BUILD_FLAGS) -o $(BINARY) $(CMD) + +## Run the full test suite with race detector +test: + go test -count=1 -race ./... + +## Smoke-test the built binary against a live target. +## Override the target with REM_LIVE_TARGET= (default: linear.app). +smoke: build + REM_LIVE_TARGET=$${REM_LIVE_TARGET:-linear.app} bash smoke.sh ./$(BINARY) + +## Install the binary into $(GOPATH)/bin/ +install: build + install -m 0755 $(BINARY) $(INSTALL_DIR)/$(BINARY) + @echo "installed → $(INSTALL_DIR)/$(BINARY)" + +## Cross-compile release binaries into dist/ +cross: + mkdir -p $(DIST) + GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -o $(DIST)/$(BINARY)-linux-amd64 $(CMD) + GOOS=darwin GOARCH=arm64 go build $(BUILD_FLAGS) -o $(DIST)/$(BINARY)-darwin-arm64 $(CMD) + @echo "cross-compiled binaries written to $(DIST)/" + +## Static analysis +lint: + go vet ./... + +## Remove compiled artifacts +clean: + rm -f $(BINARY) + rm -rf $(DIST) diff --git a/README.md b/README.md index 000b79e..cefb081 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,24 @@ rem demo From the source directory with Go 1.26 or later: ```sh -go build -trimpath -o rem ./cmd/rem +make build # or: go build -trimpath -o rem ./cmd/rem ./rem doctor ./rem linear.app --only website,seo --no-ai --out ./rem-reports/linear ./rem validate ./rem-reports/linear/evidence.json ./rem-reports/linear/report.md ./rem web ``` +### Build & test automation + +```sh +make build # Build optimized binary with -trimpath and -ldflags="-s -w" +make test # Run full test suite with race detector (-race) +make lint # Run go vet ./... +make smoke # Run end-to-end smoke test on linear.app (or $REM_LIVE_TARGET) +make install # Install binary to $(GOPATH)/bin/rem +make cross # Cross-compile for linux/amd64 and darwin/arm64 into dist/ +``` + `./rem demo` opens a recorded Cal AI investigation with no requests to source providers. Use `--no-open --port 0` to print a dynamically allocated loopback URL without launching a browser. Choose **Evidence only** in the browser to collect and view reports without a model. To compare two snapshots: diff --git a/internal/collect/seo.go b/internal/collect/seo.go index 8f8fe42..0ab4941 100644 --- a/internal/collect/seo.go +++ b/internal/collect/seo.go @@ -76,11 +76,29 @@ func SEO(ctx context.Context, get fetch.Getter, target model.Target) model.Resul if llmsProblem != "" { result.Unknowns = append(result.Unknowns, unknown("geo", "llms.txt: "+llmsProblem, llmsURL)) } + // llms.txt access policy: check whether the file allows or restricts LLM access. + var llmsAllows *bool + if llmsStatus == "present" && llmsText != "" { + lower := strings.ToLower(llmsText) + denies := strings.Contains(lower, "disallow") || strings.Contains(lower, "noai") || + strings.Contains(lower, "no-ai") || strings.Contains(lower, "forbid") || + strings.Contains(lower, "not permitted") || strings.Contains(lower, "prohibited") + grants := strings.Contains(lower, "allow") || strings.Contains(lower, "permitted") || + strings.Contains(lower, "welcome") || strings.Contains(lower, "encouraged") + v := grants && !denies + llmsAllows = &v + } if robotsStatus != "unavailable" || llmsStatus != "unavailable" { var stance []string switch llmsStatus { case "present": - stance = append(stance, "has llms.txt") + if llmsAllows != nil && !*llmsAllows { + stance = append(stance, "has llms.txt (restricts LLMs)") + } else if llmsAllows != nil && *llmsAllows { + stance = append(stance, "has llms.txt (allows LLMs)") + } else { + stance = append(stance, "has llms.txt") + } case "absent": stance = append(stance, "llms.txt absent") default: @@ -100,7 +118,7 @@ func SEO(ctx context.Context, get fetch.Getter, target model.Target) model.Resul sourceURL = llmsURL } result.Evidence = append(result.Evidence, evidence("seo", "observation", "AI-search stance: "+strings.Join(stance, ", "), sourceURL, map[string]any{ - "robots": robots, "robotsStatus": robotsStatus, "robotsURL": robotsURL, "llmsTxt": llmsText, "llmsStatus": llmsStatus, "llmsURL": llmsURL, + "robots": robots, "robotsStatus": robotsStatus, "robotsURL": robotsURL, "llmsTxt": llmsText, "llmsStatus": llmsStatus, "llmsURL": llmsURL, "llmsAllowsLLMs": llmsAllows, })) } collectSitemaps(ctx, get, base, robots, &result) diff --git a/internal/collect/tracking.go b/internal/collect/tracking.go index 4af67ed..b44bf8c 100644 --- a/internal/collect/tracking.go +++ b/internal/collect/tracking.go @@ -89,6 +89,10 @@ var signatures = []signature{ tag("loops", "Loops", "email", "lifecycle", `(?i)app\.loops\.so`, ""), tag("braze", "Braze", "email", "lifecycle", `(?i)js\.appboycdn\.com|sdk\.iad-\d+\.braze\.com`, ""), tag("iterable", "Iterable", "email", "lifecycle", `(?i)iterable\.com/api|js\.iterable\.com`, ""), + tag("amazon_ads", "Amazon Ads", "ads_pixel", "paid_amazon", `(?i)s\.amazon-adsystem\.com|amazon_advertising_tag`, ""), + tag("criteo", "Criteo", "ads_pixel", "paid_programmatic", `(?i)static\.criteo\.net/js/ld/publishertag`, ""), + tag("outbrain", "Outbrain", "ads_pixel", "paid_programmatic", `(?i)widgets\.outbrain\.com`, ""), + tag("taboola", "Taboola", "ads_pixel", "paid_programmatic", `(?i)cdn\.taboola\.com`, ""), tag("intercom", "Intercom", "chat", "", `(?i)widget\.intercom\.io|js\.intercomcdn\.com`, ""), tag("crisp", "Crisp", "chat", "", `(?i)client\.crisp\.chat`, ""), tag("drift", "Drift", "chat", "sales_led", `(?i)js\.driftt\.com`, ""), diff --git a/internal/web/app.js b/internal/web/app.js index 14be1f9..a3d8ac8 100644 --- a/internal/web/app.js +++ b/internal/web/app.js @@ -461,6 +461,21 @@ function displayRun(id, data) { const path = `/api/runs/${encodeURIComponent(id)}`; $("markdown-link").href = `${path}/report.md`; $("markdown-link").hidden = !data.report; + const copyBtn = $("copy-report-btn"); + if (copyBtn) { + copyBtn.hidden = !data.report; + copyBtn.onclick = async () => { + try { + await navigator.clipboard.writeText(data.report); + const originalText = copyBtn.textContent; + copyBtn.textContent = "Copied!"; + setTimeout(() => { copyBtn.textContent = originalText; }, 2000); + } catch (e) { + copyBtn.textContent = "Copy failed"; + setTimeout(() => { copyBtn.textContent = "Copy report"; }, 2000); + } + }; + } $("evidence-link").href = `${path}/evidence.json`; $("metrics").replaceChildren(...[ [current.evidence.length, "Evidence items"], [current.content.length, "Ads & posts"], [bundle.unknowns?.length || 0, "Unknowns"], diff --git a/internal/web/index.html b/internal/web/index.html index 2219adf..0b8401c 100644 --- a/internal/web/index.html +++ b/internal/web/index.html @@ -68,7 +68,7 @@

Investigation in progress