learncpp.py

#!/usr/bin/env python3
"""Download LearnCpp tutorial pages and render each page as a PDF."""

from __future__ import annotations

import re
import signal
import sys
import time
from pathlib import Path
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from weasyprint import HTML


BASE_URL = "https://www.learncpp.com/"
INDEX_URL = urljoin(BASE_URL, "learn-cpp-site-index/")

OUTPUT_DIR = Path("learncpp_pdfs")
HTML_DIR = OUTPUT_DIR / "html"
PDF_DIR = OUTPUT_DIR / "pdf"

REQUEST_TIMEOUT_SECONDS = 45
RENDER_TIMEOUT_SECONDS = 120


class RenderTimeout(RuntimeError):
    """Raised when one PDF render takes too long."""


def log(message: str = "") -> None:
    """Print immediately so GitHub Actions shows live progress."""

    print(message, flush=True)


def render_timeout_handler(_signum: int, _frame: object) -> None:
    raise RenderTimeout(
        f"PDF rendering exceeded {RENDER_TIMEOUT_SECONDS} seconds"
    )


def format_duration(seconds: float) -> str:
    minutes, remaining_seconds = divmod(int(seconds), 60)
    return f"{minutes}m {remaining_seconds:02d}s"


def clean_filename(value: str) -> str:
    value = " ".join(value.split())
    value = re.sub(r"[^A-Za-z0-9._ -]+", "_", value)
    return value.replace(" ", "_")[:120].strip("_")


def chapter_sort_key(chapter_number: str) -> tuple[int, ...]:
    return tuple(int(part) for part in chapter_number.split("."))


def create_session() -> requests.Session:
    retry_policy = Retry(
        total=3,
        connect=3,
        read=3,
        status=3,
        backoff_factor=1.0,
        status_forcelist=(429, 500, 502, 503, 504),
        allowed_methods=frozenset({"GET"}),
        raise_on_status=False,
    )

    session = requests.Session()
    session.headers.update(
        {
            "User-Agent": (
                "Mozilla/5.0 (X11; Linux x86_64) "
                "AppleWebKit/537.36 "
                "Chrome/126 Safari/537.36"
            )
        }
    )

    adapter = HTTPAdapter(max_retries=retry_policy)
    session.mount("https://", adapter)
    session.mount("http://", adapter)

    return session


def discover_pages(
    session: requests.Session,
) -> list[tuple[str, str, str]]:
    response = session.get(
        INDEX_URL,
        timeout=REQUEST_TIMEOUT_SECONDS,
    )
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")

    pages: list[tuple[str, str, str]] = []
    seen_urls: set[str] = set()

    for anchor in soup.select("a[href]"):
        href = anchor.get("href", "")
        title = " ".join(anchor.get_text(" ", strip=True).split())

        if "/cpp-tutorial/" not in href:
            continue

        match = re.match(r"^(\d+(?:\.\d+)?)\b", title)
        if match is None:
            continue

        url = urljoin(BASE_URL, href)
        if url in seen_urls:
            continue

        seen_urls.add(url)
        chapter_number = match.group(1)
        pages.append((chapter_number, title, url))

    pages.sort(key=lambda page: chapter_sort_key(page[0]))
    return pages


def render_page(
    session: requests.Session,
    url: str,
    html_path: Path,
    pdf_path: Path,
) -> None:
    response = session.get(
        url,
        timeout=REQUEST_TIMEOUT_SECONDS,
    )
    response.raise_for_status()

    html_path.write_text(response.text, encoding="utf-8")

    old_handler = signal.signal(
        signal.SIGALRM,
        render_timeout_handler,
    )
    signal.alarm(RENDER_TIMEOUT_SECONDS)

    try:
        HTML(
            filename=str(html_path),
            base_url=url,
        ).write_pdf(str(pdf_path))
    finally:
        signal.alarm(0)
        signal.signal(signal.SIGALRM, old_handler)

    if not pdf_path.exists():
        raise RuntimeError("PDF renderer did not create an output file")

    if pdf_path.stat().st_size == 0:
        raise RuntimeError("PDF renderer created an empty file")


def main() -> int:
    HTML_DIR.mkdir(parents=True, exist_ok=True)
    PDF_DIR.mkdir(parents=True, exist_ok=True)

    session = create_session()

    log("[index] Downloading the LearnCpp site index...")
    index_started = time.monotonic()

    try:
        pages = discover_pages(session)
    except Exception as error:
        log(f"[error] Could not download the site index: {error}")
        return 1

    index_elapsed = time.monotonic() - index_started
    total = len(pages)

    log(
        f"[index] Found {total} tutorial pages "
        f"in {format_duration(index_elapsed)}."
    )
    log()

    if total == 0:
        log(
            "[error] No tutorial pages were discovered. "
            "The website structure may have changed."
        )
        return 1

    run_started = time.monotonic()

    created = 0
    skipped = 0
    failures: list[tuple[str, str, str]] = []

    for position, (chapter_number, title, url) in enumerate(
        pages,
        start=1,
    ):
        progress = f"[{position:03d}/{total:03d}]"

        safe_title = clean_filename(title)
        html_path = HTML_DIR / f"{chapter_number}_{safe_title}.html"
        pdf_path = PDF_DIR / f"{chapter_number}_{safe_title}.pdf"

        if pdf_path.exists() and pdf_path.stat().st_size > 0:
            skipped += 1
            log(f"{progress} SKIP   {title}")
            continue

        page_started = time.monotonic()

        try:
            log(f"{progress} FETCH  {title}")
            log(f"{progress} URL    {url}")

            response = session.get(
                url,
                timeout=REQUEST_TIMEOUT_SECONDS,
            )
            response.raise_for_status()

            html_path.write_text(
                response.text,
                encoding="utf-8",
            )

            download_elapsed = time.monotonic() - page_started
            log(
                f"{progress} FETCHED in "
                f"{format_duration(download_elapsed)}"
            )

            log(f"{progress} RENDER {title}")

            old_handler = signal.signal(
                signal.SIGALRM,
                render_timeout_handler,
            )
            signal.alarm(RENDER_TIMEOUT_SECONDS)

            try:
                HTML(
                    filename=str(html_path),
                    base_url=url,
                ).write_pdf(str(pdf_path))
            finally:
                signal.alarm(0)
                signal.signal(signal.SIGALRM, old_handler)

            if not pdf_path.exists() or pdf_path.stat().st_size == 0:
                raise RuntimeError(
                    "PDF renderer produced no usable output"
                )

            created += 1

            page_elapsed = time.monotonic() - page_started
            run_elapsed = time.monotonic() - run_started

            processed = created + skipped + len(failures)
            average_seconds = run_elapsed / max(processed, 1)
            estimated_remaining = average_seconds * (total - position)

            size_kib = pdf_path.stat().st_size / 1024

            log(
                f"{progress} DONE   {title} "
                f"({size_kib:.1f} KiB, "
                f"{format_duration(page_elapsed)}, "
                f"ETA {format_duration(estimated_remaining)})"
            )

        except Exception as error:
            if pdf_path.exists():
                pdf_path.unlink()

            message = str(error) or repr(error)
            failures.append((title, url, message))

            log(f"{progress} FAILED {title}")
            log(f"{progress} ERROR  {message}")

        log()

    total_elapsed = time.monotonic() - run_started

    log("=" * 72)
    log(
        f"[summary] Finished in {format_duration(total_elapsed)}"
    )
    log(f"[summary] Created: {created}")
    log(f"[summary] Skipped: {skipped}")
    log(f"[summary] Failed:  {len(failures)}")

    if failures:
        log()
        log("[summary] Failed pages:")

        for title, url, message in failures:
            log(f"  - {title}")
            log(f"    URL: {url}")
            log(f"    Error: {message}")

    if created + skipped == 0:
        return 1

    return 0


if __name__ == "__main__":
    sys.exit(main())