build_docs.py

#!/usr/bin/env python3
"""
EDS 217 Documentation Build Script (Python version)
This script builds the Quarto website and prepares it for GitHub Pages deployment
Supports incremental builds by default (only changed files) with --full option for complete rebuild
"""

import os
import sys
import subprocess
import shutil
import argparse
import time
import glob
import re
from pathlib import Path

def run_command(command, description):
    """Run a shell command and handle errors."""
    print(f"🔨 {description}...")
    try:
        result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
        return result
    except subprocess.CalledProcessError as e:
        print(f"❌ Error: {description} failed")
        print(f"   Command: {command}")
        print(f"   Error: {e.stderr}")
        sys.exit(1)

# Files that shape every page. A change to the navbar, the theme or the shared
# CSS is not visible in any single .qmd, so a page whose own source is untouched
# is still out of date once one of these moves.
SITE_WIDE = ["_quarto.yml", "meds-website-styles.scss",
             "course-materials/assets/css/exercises.css"]


def _output_for(src):
    """The rendered HTML that a source file produces, under docs/."""
    if src.endswith(".qmd"):
        return Path("docs") / (src[:-4] + ".html")
    if src.endswith(".ipynb"):
        return Path("docs") / (src[:-6] + ".html")
    return None


def get_changed_files():
    """Every source whose rendered output is missing or older than the source.

    This used to ask git which files differed from HEAD, which is a different
    question and the wrong one. Committing before rendering leaves a clean
    working tree, so `git diff HEAD` returned nothing and the build reported
    success having built nothing. It happened twice on 2026-08-27 and 28: once
    leaving twenty pages stale, once leaving one. Both reported success, and
    publish.sh could not see it because d5_render_complete counts that ninety
    HTML files exist rather than that they are current.

    Staleness is a property of the pair (source, output), so compare the two.
    The one-second tolerance matches d10_render_current, which asks the same
    question from the other side and is what caught the bug.
    """
    try:
        sources = get_all_buildable_files()
    except Exception as e:
        print(f"⚠️  Error listing buildable files: {e}")
        print("   Falling back to full build")
        return None

    docs = Path("docs")
    if not docs.exists():
        print("📁 docs/ does not exist yet, so every page needs building")
        return None

    # A site-wide input newer than any output invalidates the whole site.
    newest_site_wide = 0.0
    which = None
    for f in SITE_WIDE:
        p = Path(f)
        if p.exists() and p.stat().st_mtime > newest_site_wide:
            newest_site_wide, which = p.stat().st_mtime, f

    changed, missing = [], 0
    site_wide_hits = 0
    for src in sources:
        out = _output_for(src)
        if out is None:
            continue
        if not out.exists():
            changed.append(src)
            missing += 1
            continue
        out_mtime = out.stat().st_mtime
        if Path(src).stat().st_mtime > out_mtime + 1:
            changed.append(src)
        elif newest_site_wide > out_mtime + 1:
            changed.append(src)
            site_wide_hits += 1

    if changed:
        print(f"🔍 {len(changed)} page(s) need building:")
        if missing:
            print(f"   {missing} never rendered")
        if site_wide_hits:
            print(f"   {site_wide_hits} older than {which}, which is on every page")
        for s in changed[:10]:
            print(f"   {s}")
        if len(changed) > 10:
            print(f"   ... and {len(changed) - 10} more")
    return changed


def _render_exclusions():
    """The paths _quarto.yml tells quarto not to render.

    Quarto builds every .qmd and .ipynb in the tree except the entries listed
    with a leading "!" in the project render block. Reading them here is what
    keeps a full build from rendering retired 2025 pages back onto the site.
    """
    patterns, inside, indent = [], False, None
    for line in Path("_quarto.yml").read_text().splitlines():
        stripped = line.strip()
        if stripped == "render:":
            inside = True
            indent = len(line) - len(line.lstrip())
            continue
        if inside:
            current = len(line) - len(line.lstrip())
            if stripped and not stripped.startswith("#") and current <= indent:
                break
            m = re.match(r'^-\s*"?([^"#]+?)"?\s*$', stripped)
            if m:
                patterns.append(m.group(1))
    return [p[1:] for p in patterns if p.startswith("!")]


def _matches_glob(path, pattern):
    """Match a path against a quarto render pattern, including ** segments."""
    rx = re.escape(pattern)
    rx = rx.replace(r"\*\*/", "(?:.*/)?").replace(r"\*\*", ".*")
    rx = rx.replace(r"\*", "[^/]*").replace(r"\?", ".")
    return re.fullmatch(rx, path) is not None


# Directories that are never part of the site whatever _quarto.yml says.
# _to_delete/ holds retired files awaiting manual removal, and the checkpoint
# directories hold stale copies Jupyter writes.
NEVER_BUILD = {"_to_delete", ".ipynb_checkpoints", "docs", ".quarto", "__pycache__"}


def get_all_buildable_files():
    """Every file quarto renders, per the render block in _quarto.yml."""
    excluded = _render_exclusions()
    all_files = []
    for pattern in ("*.qmd", "*.ipynb"):
        for path in Path(".").rglob(pattern):
            rel = str(path)
            if NEVER_BUILD & set(path.parts):
                continue
            if any(_matches_glob(rel, ex) for ex in excluded):
                continue
            all_files.append(rel)
    return sorted(all_files)


def activate_conda_environment():
    """Activate the eds217_2026 conda environment."""
    print("🐍 Activating eds217_2026 environment...")
    
    # Get conda base path
    try:
        result = subprocess.run(["conda", "info", "--base"], 
                              check=True, capture_output=True, text=True)
        conda_base = result.stdout.strip()
    except (subprocess.CalledProcessError, FileNotFoundError):
        print("❌ Error: Conda not found. Please install conda/miniconda/mambaforge")
        sys.exit(1)
    
    # Set up environment variables for conda
    conda_sh = os.path.join(conda_base, "etc", "profile.d", "conda.sh")
    if not os.path.exists(conda_sh):
        print(f"❌ Error: Conda script not found at {conda_sh}")
        sys.exit(1)
    
    # Update PATH to include the eds217_2026 environment
    try:
        result = subprocess.run(["conda", "info", "--envs"], 
                              check=True, capture_output=True, text=True)
        
        # Find eds217_2026 environment path
        env_path = None
        for line in result.stdout.split('\n'):
            if 'eds217_2026' in line:
                parts = line.split()
                if len(parts) >= 2:
                    env_path = parts[-1]  # Last part is the path
                    break
        
        if not env_path:
            print("❌ Error: eds217_2026 environment not found")
            print("Please create the environment first:")
            print("   conda env list")
            sys.exit(1)
        
        # Add environment bin to PATH
        env_bin = os.path.join(env_path, "bin")
        current_path = os.environ.get("PATH", "")
        os.environ["PATH"] = f"{env_bin}:{current_path}"
        os.environ["CONDA_DEFAULT_ENV"] = "eds217_2026"
        os.environ["CONDA_PREFIX"] = env_path
        
        print(f"   ✅ Environment activated: eds217_2026")
        
    except subprocess.CalledProcessError:
        print("❌ Error: Failed to get conda environment information")
        sys.exit(1)

def check_prerequisites():
    """Check if required tools are available."""
    print("🔍 Checking prerequisites...")
    
    # Check if quarto is installed
    try:
        subprocess.run(["quarto", "--version"], check=True, capture_output=True)
        print("   ✅ Quarto found")
    except (subprocess.CalledProcessError, FileNotFoundError):
        print("❌ Error: Quarto is not installed or not in PATH")
        print("Please install Quarto from https://quarto.org/docs/get-started/")
        sys.exit(1)
    
    # Check if we're in the right directory
    if not Path("_quarto.yml").exists():
        print("❌ Error: _quarto.yml not found. Are you in the project root directory?")
        sys.exit(1)
    print("   ✅ _quarto.yml found")

def ensure_docs_intact():
    """An incremental build adds to docs/. It must never be the thing that empties it.

    clean_docs() used to run on every path, so building one changed page deleted
    the whole rendered site and replaced it with that page. If docs/ is already
    incomplete when an incremental build starts, an earlier run left it that way
    and rendering on top of it would publish the gap. Stop and say so.
    """
    docs_path = Path("docs")
    docs_path.mkdir(exist_ok=True)
    pages = list(docs_path.rglob("*.html"))
    if len(pages) < 50:
        print(f"\u274c docs/ holds only {len(pages)} rendered page(s).")
        print("   An incremental build would leave the published site incomplete.")
        print("   Run a full rebuild instead:  python build_docs.py --full")
        sys.exit(1)
    print(f"\U0001f4c1 docs/ holds {len(pages)} rendered pages; adding to them.")


def clean_docs():
    """Clean the docs directory and any stray HTML files."""
    print("🧹 Cleaning previous build...")
    
    # Remove any stray HTML files in root directory
    for html_file in Path(".").glob("*.html"):
        html_file.unlink()
    print("   Removed any stray HTML files from root")
    
    # Clean the docs directory
    docs_path = Path("docs")
    if docs_path.exists():
        shutil.rmtree(docs_path)
        print("   Previous build files removed from docs/")
    
    # Ensure docs directory exists
    docs_path.mkdir(exist_ok=True)
    print("   Created docs/ directory")

def clean_intermediate_files():
    """Remove all HTML and other intermediate files from course_materials directories."""
    print("🧹 Cleaning intermediate files from course_materials...")
    
    # Extensions to clean up
    extensions_to_remove = [
        "*.html",           # Rendered HTML files
        "*_files/",         # Quarto output directories
        "*.ipynb_checkpoints/",  # Jupyter checkpoints
        "*/.quarto/",       # Quarto cache directories
        "*.aux",            # LaTeX auxiliary files
        "*.log",            # Log files
        "*.out",            # LaTeX output files
        "*.toc",            # Table of contents files
        "*.nav",            # LaTeX navigation files
        "*.snm",            # LaTeX slide navigation files
        "*.fls",            # LaTeX file list
        "*.fdb_latexmk",    # LaTeX make files
        "*.synctex.gz",     # SyncTeX files
    ]
    
    course_materials_path = Path("course-materials")
    if not course_materials_path.exists():
        print("   No course-materials directory found")
        return
    
    total_removed = 0
    
    for extension in extensions_to_remove:
        if extension.endswith("/"):
            # Remove directories
            for item in course_materials_path.rglob(extension):
                if item.is_dir():
                    shutil.rmtree(item)
                    total_removed += 1
                    print(f"   Removed directory: {item}")
        else:
            # Remove files
            for item in course_materials_path.rglob(extension):
                if item.is_file():
                    item.unlink()
                    total_removed += 1
                    print(f"   Removed file: {item}")
    
    print(f"   ✅ Cleaned {total_removed} intermediate files/directories")

def build_site(files_to_build=None, full_build=False):
    """Build the Quarto website with detailed progress tracking."""
    import time
    
    # Determine which files to build
    if full_build or not files_to_build:
        print("📊 Analyzing files to build...")
        all_files = get_all_buildable_files()
        files_to_process = all_files
        build_type = "full build"
    else:
        files_to_process = files_to_build if files_to_build else []
        build_type = "incremental build"
    
    if not files_to_process:
        print("✅ No files to build - all files are up to date!")
        return
    
    total_files = len(files_to_process)
    print(f"🚀 Starting {build_type} ({total_files} file{' ' if total_files == 1 else 's'} to process)")
    print("")
    
    # Track timing
    overall_start_time = time.time()
    file_times = []
    
    # Build each file with detailed progress
    for i, file_path in enumerate(files_to_process, 1):
        file_start_time = time.time()
        
        # Calculate progress
        progress_pct = (i - 1) / total_files * 100
        remaining_files = total_files - (i - 1)
        
        # Estimate time remaining based on average file time
        if file_times:
            avg_time_per_file = sum(file_times) / len(file_times)
            estimated_remaining = avg_time_per_file * remaining_files
            eta_str = f" (ETA: {estimated_remaining:.1f}s)"
        else:
            eta_str = ""
        
        # Show progress header
        elapsed = time.time() - overall_start_time
        print(f"[{i:2d}/{total_files}] ({progress_pct:5.1f}%) Building: {file_path}")
        print(f"        ⏱️  Elapsed: {elapsed:.1f}s{eta_str}")
        
        # Build the file with a simpler progress indicator
        print(f"        🔨 Rendering... ", end="", flush=True)
        
        try:
            # Run the command
            result = subprocess.run(
                f"quarto render '{file_path}'", 
                shell=True, 
                check=True, 
                capture_output=True, 
                text=True
            )
            
            file_elapsed = time.time() - file_start_time
            file_times.append(file_elapsed)
            
            print(f"✅ Done ({file_elapsed:.1f}s)")
            
        except subprocess.CalledProcessError as e:
            file_elapsed = time.time() - file_start_time
            print(f"❌ Failed ({file_elapsed:.1f}s)")
            print(f"        Error: {e.stderr}")
            print(f"        Command: quarto render '{file_path}'")
            sys.exit(1)
        
        # Add spacing between files (except for the last one)
        if i < total_files:
            print("")
    
    # Final summary
    total_elapsed = time.time() - overall_start_time
    avg_time = total_elapsed / total_files if total_files > 0 else 0
    
    print("")
    print("📊 Build completed!")
    print(f"   ⏱️  Total time: {total_elapsed:.1f}s")
    print(f"   📈 Average per file: {avg_time:.1f}s")
    print(f"   📁 Files processed: {total_files}")
    
    if file_times:
        fastest = min(file_times)
        slowest = max(file_times)
        print(f"   ⚡ Fastest file: {fastest:.1f}s")
        print(f"   🐌 Slowest file: {slowest:.1f}s")

def verify_build():
    """Verify the build was successful."""
    docs_path = Path("docs")
    
    if not docs_path.exists() or not any(docs_path.iterdir()):
        print("❌ Error: docs directory is empty or doesn't exist after build")
        sys.exit(1)
    
    print("✅ Build completed successfully!")
    print("📁 Documentation built in docs/ directory")
    
    # Count files and get size
    file_count = len(list(docs_path.rglob("*")))
    size_result = subprocess.run(["du", "-sh", "docs"], capture_output=True, text=True)
    size = size_result.stdout.split()[0] if size_result.returncode == 0 else "unknown"
    
    print("📊 Build summary:")
    print(f"   - Files in docs/: {file_count}")
    print(f"   - Size: {size}")
    
    # Check critical files
    index_html = docs_path / "index.html"
    nojekyll = docs_path / ".nojekyll"
    
    if index_html.exists():
        print("   - ✅ index.html found")
    else:
        print("   - ⚠️  index.html not found")
    
    if nojekyll.exists():
        print("   - ✅ .nojekyll found (GitHub Pages compatibility)")
    else:
        print("   - ⚠️  .nojekyll not found")

def parse_arguments():
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(
        description="Build EDS 217 documentation with incremental build support"
    )
    parser.add_argument(
        "--serve", "-s",
        action="store_true",
        help="Start local server after building"
    )
    parser.add_argument(
        "--full", "-f",
        action="store_true",
        help="Force full rebuild of all files (default: incremental build)"
    )
    parser.add_argument(
        "--clean", "-c",
        action="store_true",
        help="Clean intermediate files (HTML, _files/, etc.) from course_materials directories"
    )
    return parser.parse_args()

def serve_locally():
    """Start local server to preview the site."""
    print("🌐 Starting local server...")
    print("   Press Ctrl+C to stop the server when done")
    print("   Your site will open in your default browser")
    print("")
    
    try:
        subprocess.run(["quarto", "preview"], check=True)
    except KeyboardInterrupt:
        print("\n🛑 Server stopped")
    except subprocess.CalledProcessError as e:
        print(f"❌ Error starting server: {e}")
        sys.exit(1)

def main():
    """Main build process."""
    args = parse_arguments()
    
    # Handle clean-only operation
    if args.clean and not any([args.full, args.serve]):
        # Check if this is a clean-only operation (no other build flags)
        print("🧹 Cleaning intermediate files...")
        
        # We still need to check prerequisites for conda environment
        activate_conda_environment()
        check_prerequisites()
        
        clean_intermediate_files()
        print("✅ Intermediate files cleaned successfully!")
        return
    
    if args.full:
        print("🚀 Starting EDS 217 documentation build (FULL BUILD)...")
    else:
        print("🚀 Starting EDS 217 documentation build (incremental)...")
    
    try:
        activate_conda_environment()
        check_prerequisites()
        
        files_to_build = None
        if not args.full:
            # Determine which files need to be built
            changed_files = get_changed_files()
            if changed_files is not None:
                files_to_build = changed_files
                if not files_to_build:
                    print("✅ Every page in docs/ is newer than its source - skipping build")
                    print("   Use --full flag to force a complete rebuild")
                    if args.clean:
                        clean_intermediate_files()
                    return
        
        if args.clean:
            clean_intermediate_files()

        if args.full:
            clean_docs()
        else:
            ensure_docs_intact()
        build_site(files_to_build, args.full)
        verify_build()
        
        print("")
        if args.full or (files_to_build is None):
            print("🎉 Your site is ready for deployment!")
        else:
            print(f"🎉 Successfully built {len(files_to_build)} changed file(s)!")
        
        if args.serve:
            serve_locally()
        else:
            print("📝 Next steps:")
            print("   1. Review the built site in the docs/ folder")
            print("   2. Commit and push changes to your repository")
            print("   3. Your GitHub Pages site will update automatically")
            print("")
            print("🌐 To preview locally, you can run:")
            print("   quarto preview")
            print("   Or use: python build_docs.py --serve")
            if not args.full:
                print("")
                print("💡 Tip: Use --full flag for complete rebuild when needed")
        
    except KeyboardInterrupt:
        print("\n❌ Build cancelled by user")
        sys.exit(1)
    except Exception as e:
        print(f"❌ Unexpected error: {e}")
        sys.exit(1)

if __name__ == "__main__":
    main()