diff --git a/.github/scripts/generate-preview-index.sh b/.github/scripts/generate-preview-index.sh deleted file mode 100755 index bbc3cf84..00000000 --- a/.github/scripts/generate-preview-index.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash -set -e # Exit on any error - -# Generate the preview index page -TEMPLATE_FILE=".github/templates/preview-index.html" -OUTPUT_FILE="index.html" - -# Validate template file exists -if [ ! -f "$TEMPLATE_FILE" ]; then - echo "ERROR: Template file $TEMPLATE_FILE not found!" - echo "This indicates a problem with the workflow setup." - exit 1 -fi - -# Start with the template -if ! cp "$TEMPLATE_FILE" "$OUTPUT_FILE"; then - echo "ERROR: Failed to copy template file to $OUTPUT_FILE" - exit 1 -fi - -# Generate preview items HTML -PREVIEW_ITEMS="" -FOUND_PREVIEWS=false -PREVIEW_COUNT=0 - -for dir in pr-*/; do - if [ -d "$dir" ] && [ "$dir" != "pr-preview/" ]; then - # Validate directory contains expected files - if [ ! -f "$dir/index.html" ]; then - echo "Warning: $dir appears to be incomplete (no index.html), skipping..." - continue - fi - - # Extract PR number from directory name (handles both old pr-123 and new pr-123-title formats) - pr_num=$(basename "$dir" | sed 's/pr-\([0-9][0-9]*\).*/\1/') - # Validate PR number is numeric - if ! [[ "$pr_num" =~ ^[0-9]+$ ]]; then - echo "Warning: $dir has invalid PR number format, skipping..." - continue - fi - - # Extract title part if it exists (for display purposes) - title_part=$(basename "$dir" | sed 's/pr-[0-9][0-9]*-\(.*\)/\1/' | sed 's/-/ /g') - if [ "$title_part" = "$(basename "$dir")" ]; then - # No title part found, use empty string - title_part="" - fi - - # Additional validation: PR number must be reasonable range - if [ "$pr_num" -lt 1 ] || [ "$pr_num" -gt 99999 ]; then - echo "Warning: $dir has PR number out of valid range, skipping..." - continue - fi - - # HTML escape the directory name to prevent injection - dir_escaped=$(echo "$dir" | sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"'"'/\'/g') - pr_num_escaped=$(echo "$pr_num" | sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"'"'/\'/g') - title_part_escaped=$(echo "$title_part" | sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"'"'/\'/g') - - # Create display title - if [ -n "$title_part" ]; then - display_title="PR #$pr_num_escaped: $title_part_escaped" - else - display_title="PR #$pr_num_escaped Preview" - fi - - PREVIEW_ITEMS+="
  • " - PREVIEW_ITEMS+="$display_title" - PREVIEW_ITEMS+="
    https://preview.wafer.space/$dir_escaped
    " - PREVIEW_ITEMS+="
  • " - FOUND_PREVIEWS=true - PREVIEW_COUNT=$((PREVIEW_COUNT + 1)) - fi -done - -# If no previews found, show empty state -if [ "$FOUND_PREVIEWS" = false ]; then - PREVIEW_ITEMS="" -fi - -# Replace placeholder with actual items -if ! sed -i "s||$PREVIEW_ITEMS|g" "$OUTPUT_FILE"; then - echo "ERROR: Failed to replace placeholder in template" - echo "Check template file format and placeholder existence" - exit 1 -fi - -# Validate output file was created successfully -if [ ! -f "$OUTPUT_FILE" ] || [ ! -s "$OUTPUT_FILE" ]; then - echo "ERROR: Output file $OUTPUT_FILE was not created or is empty" - exit 1 -fi - -echo "Generated preview index with $PREVIEW_COUNT active previews" \ No newline at end of file diff --git a/.github/scripts/generate-preview-root.py b/.github/scripts/generate-preview-root.py new file mode 100755 index 00000000..a41393bf --- /dev/null +++ b/.github/scripts/generate-preview-root.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Generate the root of preview.wafer.space: index.html and redirects.json. + +Run from the root of the preview.wafer.space repository, just before the +commit that will hold the result (pr-preview.yml and cleanup-previews.yml run +it after adding or removing previews, and pr-preview.yml again after merging +in another deploy's push). Both files are built first and only then written, +so a failure leaves the previous pair in place. The workflows run the base +branch's copy of this script and its template, not a pull request's. + +A preview is a pr-NNN directory with an index.html, or its pr-NNN-title +alias; both files only cover those. + +index.html lists every preview, from .github/templates/preview-index.html +next to this script. + +redirects.json is the body of Cloudflare's "replace list items" call for the +preview_wafer_space Bulk Redirect list, which the redirects sync Worker +(https://github.com/wafer-space/cloudflare-redirects-sync) loads as is, so a +bad entry would stop the whole list loading. Each pr-NNN preview's +redirects.json (written by _plugins/redirect_list.rb for the production +hostname) is moved under the preview, keeping only the fields this sets: + +- source wafer.space/ becomes preview.wafer.space/pr-NNN/, and + a source already seen in that preview is skipped; +- a target on https://wafer.space follows it under + https://preview.wafer.space/pr-NNN/, any other absolute http(s) target is + left as is, and an entry with any other target is skipped; +- the status becomes 302, because browsers cache a 301 indefinitely and a + preview's redirects change with every push; +- the preview's own revision item (wafer.space/revision) is left out. + +Its first item records the revision, as the Worker requires: +preview.wafer.space/revision redirects to PREVIEW_SOURCE_COMMIT (the site +commit whose deploy or cleanup wrote this), and its comment is the git +describe number of the preview repository commit that will hold the file +(HEAD's plus one). Because each run commits straight after this, including +after merging in another deploy's push, the numbers only go up. They count +commits since the v0.0 tag, so the preview repository's history must not be +rewritten and no other v* tag added, or the numbers would drop and the +Worker would stop loading the list until it is forced to. +""" + +import html +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +TEMPLATE = Path(__file__).resolve().parent.parent / "templates" / "preview-index.html" +PLACEHOLDER = "" +HIDDEN_EMPTY_STATE = 'id="empty-state" style="display: none;"' + +PRODUCTION_HOST = "wafer.space" +PREVIEW_HOST = "preview.wafer.space" +COMMIT_URL = "https://github.com/wafer-space/wafer-space.github.io/commit/" +# pr-123, or its pr-123-some-title alias +LISTED_DIR = re.compile(r"^pr-([0-9]+)(?:-(.*))?$") +TARGET = re.compile(r"^https://wafer\.space(?=$|[/?#])/?(.*)$", re.S) +ABSOLUTE = re.compile(r"^https?://[^/?#\s]+", re.I) +# The fields of a redirect this writes, besides source_url and target_url +KEPT_FIELDS = ("preserve_query_string", "preserve_path_suffix") +# Cloudflare's Free plan allows 10,000 redirects across all lists +WARN_TOTAL = 9000 + + +def previews(directories): + """(directory, PR number, title) for each preview, in PR number order.""" + found = [] + for directory in directories: + match = LISTED_DIR.match(directory.name) + if not match: + continue + if not (directory / "index.html").is_file(): + print(f"Warning: {directory.name}/ appears to be incomplete (no index.html), skipping...") + continue + number, title = int(match.group(1)), (match.group(2) or "").replace("-", " ") + if not 1 <= number <= 99999: + print(f"Warning: {directory.name}/ has PR number out of valid range, skipping...") + continue + found.append((directory, number, title)) + return sorted(found, key=lambda preview: (preview[1], preview[0].name)) + + +def index_html(found): + """The preview index page.""" + items = [] + for directory, number, title in found: + name = html.escape(f"{directory.name}/") + label = f"PR #{number}: {html.escape(title)}" if title else f"PR #{number} Preview" + items.append(f'
  • {label}' + f'
    https://{PREVIEW_HOST}/{name}
  • ') + page = TEMPLATE.read_text() + for marker in (PLACEHOLDER, HIDDEN_EMPTY_STATE): + if marker not in page: + raise ValueError(f"{TEMPLATE} has no {marker}") + if not items: + page = page.replace(HIDDEN_EMPTY_STATE, 'id="empty-state"') + return page.replace(PLACEHOLDER, "".join(items)) + + +def next_revision(): + """git describe for the commit that will hold the file, without its hash.""" + try: + describe = subprocess.run( + ["git", "describe", "--tags", "--long", "--match", "v[0-9]*", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + except subprocess.CalledProcessError as err: + print(f"ERROR: git describe failed: {err.stderr.strip()}") + raise + match = re.fullmatch(r"(.+)-([0-9]+)-g[0-9a-f]+", describe) + if not match: + raise ValueError(f"unexpected git describe output {describe!r}") + return f"{match.group(1)}-{int(match.group(2)) + 1}" + + +def preview_items(directory): + """The preview's redirects, moved under it, or [] with a warning.""" + path = directory / "redirects.json" + try: + data = json.loads(path.read_text()) + except (OSError, ValueError) as err: + print(f"Warning: skipping {path}: {err}") + return [] + if not isinstance(data, list): + print(f"Warning: skipping {path}: not a JSON array") + return [] + items = [] + seen = set() + for item in data: + redirect = item.get("redirect") if isinstance(item, dict) else None + source = redirect.get("source_url") if isinstance(redirect, dict) else None + target = redirect.get("target_url") if isinstance(redirect, dict) else None + if (not isinstance(source, str) or not source.startswith(f"{PRODUCTION_HOST}/") + or not isinstance(target, str) or not ABSOLUTE.match(target)): + print(f"Warning: skipping an entry of {path}: {json.dumps(item)[:200]}") + continue + if source == f"{PRODUCTION_HOST}/revision": + continue + if source in seen: + print(f"Warning: skipping a second entry for {source} in {path}") + continue + seen.add(source) + match = TARGET.match(target) + comment = item.get("comment") + items.append({ + **({"comment": comment} if isinstance(comment, str) else {}), + "redirect": { + "source_url": f"{PREVIEW_HOST}/{directory.name}/{source[len(PRODUCTION_HOST) + 1:]}", + "target_url": f"https://{PREVIEW_HOST}/{directory.name}/{match.group(1)}" if match else target, + "status_code": 302, + **{key: redirect[key] for key in KEPT_FIELDS if isinstance(redirect.get(key), bool)}, + }, + }) + return items + + +def redirects_json(found, commit): + """The preview list, starting with its revision item, and how many previews it covers.""" + items = [] + covered = 0 + for directory, _, title in found: + # The pr-NNN-title aliases only redirect to their pr-NNN directory + if not title and (directory / "redirects.json").is_file(): + moved = preview_items(directory) + covered += bool(moved) + items += moved + items.sort(key=lambda item: item["redirect"]["source_url"]) + revision = next_revision() + items.insert(0, { + "comment": f"{revision} {commit[:7]}", + "redirect": { + "source_url": f"{PREVIEW_HOST}/revision", + "target_url": f"{COMMIT_URL}{commit}", + "status_code": 302, + }, + }) + return items, covered, revision + + +def main(): + commit = os.environ.get("PREVIEW_SOURCE_COMMIT", "") + if not re.fullmatch(r"[0-9a-f]{40}", commit): + print("ERROR: PREVIEW_SOURCE_COMMIT must be the full hash of the site commit being deployed") + return 1 + found = previews(d for d in Path(".").iterdir() if d.is_dir()) + page = index_html(found) + items, covered, revision = redirects_json(found, commit) + + Path("index.html").write_text(page) + Path("redirects.json").write_text(json.dumps(items, indent=2) + "\n") + print(f"Generated preview index with {len(found)} active previews") + print(f"Wrote redirects.json at {revision}: {len(items)} entries from {covered} preview(s)") + if len(items) > WARN_TOTAL: + print(f"::warning::The preview redirect list has {len(items)} entries; Cloudflare allows 10,000 across all lists") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/cleanup-previews.yml b/.github/workflows/cleanup-previews.yml index a7130a17..7c1ce39d 100644 --- a/.github/workflows/cleanup-previews.yml +++ b/.github/workflows/cleanup-previews.yml @@ -137,6 +137,9 @@ jobs: - name: Perform cleanup of closed PR previews if: steps.find-cleanup.outputs.cleanup_needed == 'true' + env: + # Recorded in the preview redirects.json's revision item + PREVIEW_SOURCE_COMMIT: ${{ github.sha }} run: | cd preview-repo @@ -170,12 +173,10 @@ jobs: find . -maxdepth 1 -type d -name "pr-${PR_NUMBER}-*" -exec rm -rf {} + || true done - # Copy script and template, then regenerate index - cp ../.github/scripts/generate-preview-index.sh ./ - mkdir -p .github/templates - cp ../.github/templates/preview-index.html .github/templates/ - chmod +x generate-preview-index.sh - ./generate-preview-index.sh + # Regenerate the preview index and redirects.json, removing what + # the shell script this replaced left in the preview repository + rm -rf generate-preview-index.sh .github/templates + python3 ../.github/scripts/generate-preview-root.py # Commit changes if any git add . diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml index 3dbdc37b..c724a27e 100644 --- a/.github/workflows/pr-preview.yml +++ b/.github/workflows/pr-preview.yml @@ -94,7 +94,24 @@ jobs: # Copy built site and workflow files to temp location before switching branches cp -r _site /tmp/pr-site-${{ steps.pr-directory.outputs.directory_name }} cp -r .github /tmp/pr-github-${{ steps.pr-directory.outputs.directory_name }} - + + - name: Get the preview index generator from the base branch + run: | + # The generator writes the preview site's shared index.html and + # redirects.json (loaded into Cloudflare), so run the base branch's + # copy, not this pull request's + TOOLS="/tmp/pr-tools-${{ steps.pr-directory.outputs.directory_name }}" + mkdir -p "$TOOLS/scripts" "$TOOLS/templates" + git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.base.sha }}" + if git cat-file -e FETCH_HEAD:.github/scripts/generate-preview-root.py; then + git show FETCH_HEAD:.github/scripts/generate-preview-root.py > "$TOOLS/scripts/generate-preview-root.py" + git show FETCH_HEAD:.github/templates/preview-index.html > "$TOOLS/templates/preview-index.html" + else + echo "The base branch has no generate-preview-root.py yet; using this pull request's" + cp .github/scripts/generate-preview-root.py "$TOOLS/scripts/" + cp .github/templates/preview-index.html "$TOOLS/templates/" + fi + - name: Setup SSH and clone preview repository run: | # Setup SSH for deploying to dedicated preview repository (memory-only) @@ -142,24 +159,15 @@ jobs: echo "preview.wafer.space" > CNAME fi - - name: Create index page for preview listing + - name: Generate the preview index and redirects.json + env: + # Recorded in the preview redirects.json's revision item + PREVIEW_SOURCE_COMMIT: ${{ github.event.pull_request.head.sha }} run: | cd preview-repo - # Copy script and template from temp location, then generate preview index - if [ ! -f "/tmp/pr-github-${{ steps.pr-directory.outputs.directory_name }}/scripts/generate-preview-index.sh" ]; then - echo "ERROR: Preview generation script not found in temp location" - exit 1 - fi - if [ ! -f "/tmp/pr-github-${{ steps.pr-directory.outputs.directory_name }}/templates/preview-index.html" ]; then - echo "ERROR: Preview template not found in temp location" - exit 1 - fi - - cp "/tmp/pr-github-${{ steps.pr-directory.outputs.directory_name }}/scripts/generate-preview-index.sh" ./ - mkdir -p .github/templates - cp "/tmp/pr-github-${{ steps.pr-directory.outputs.directory_name }}/templates/preview-index.html" .github/templates/ - chmod +x generate-preview-index.sh - ./generate-preview-index.sh + # Left in the preview repository by the shell script this replaced + rm -rf generate-preview-index.sh .github/templates + python3 "/tmp/pr-tools-${{ steps.pr-directory.outputs.directory_name }}/scripts/generate-preview-root.py" - name: Deploy to preview repository run: |