Main.py

from __future__ import annotations

import logging
from contextlib import closing
from pathlib import Path
from typing import Optional, Set

import requests
from bs4 import BeautifulSoup

# Configure these values before running the script.
STARTING_URL = "https://example.com/first-post"
ENDING_URL = "https://example.com/final-post"
OUTPUT_FILE = "webserial.html"

REQUEST_TIMEOUT_SECONDS = 30
ENTRY_CONTENT_CLASS = "entry-content"
ENTRY_TITLE_CLASS = "entry-title"

LOGGER = logging.getLogger(__name__)


def configure_logging() -> None:
    logging.basicConfig(level=logging.INFO, format="%(message)s")


def validate_configuration() -> None:
    placeholders = {
        "STARTING_URL": STARTING_URL,
        "ENDING_URL": ENDING_URL,
        "OUTPUT_FILE": OUTPUT_FILE,
    }

    for field_name, value in placeholders.items():
        if not value.strip():
            raise ValueError(f"{field_name} must not be empty.")

    if "example.com" in STARTING_URL or "example.com" in ENDING_URL:
        raise ValueError("Replace STARTING_URL and ENDING_URL with real post URLs before running the scraper.")


def log_error(message: str) -> None:
    LOGGER.error(message)


def is_good_response(response: requests.Response) -> bool:
    content_type = response.headers.get("Content-Type", "").lower()
    return response.status_code == 200 and "html" in content_type


def simple_get(url: str, session: requests.Session) -> Optional[bytes]:
    try:
        with closing(
            session.get(
                url,
                stream=True,
                timeout=REQUEST_TIMEOUT_SECONDS,
            )
        ) as response:
            if is_good_response(response):
                return response.content

            log_error(
                f"Request to {url} returned status {response.status_code} with content type "
                f"{response.headers.get('Content-Type', 'unknown')}."
            )
            return None
    except requests.RequestException as exc:
        log_error(f"Error during request to {url}: {exc}")
        return None


def get_soup(url: str, session: requests.Session) -> BeautifulSoup:
    raw_html = simple_get(url, session)
    if raw_html is None:
        raise RuntimeError(f"Could not retrieve HTML content from {url}.")

    return BeautifulSoup(raw_html, "html.parser")


def get_next_url(html: BeautifulSoup) -> Optional[str]:
    link = html.find("link", {"rel": "next"})
    if link is None:
        return None

    return link.get("href")


def get_chapter(html: BeautifulSoup, url: str) -> str:
    title = html.find("h1", {"class": ENTRY_TITLE_CLASS})
    if title is None:
        raise RuntimeError(f"Could not find a chapter title on {url}.")

    content = html.find("div", {"class": ENTRY_CONTENT_CLASS})
    if content is None:
        raise RuntimeError(f"Could not find the chapter body on {url}.")

    paragraphs = content.find_all("p")
    if not paragraphs:
        raise RuntimeError(f"Could not find paragraph content on {url}.")

    chapter_body = "".join(str(paragraph) for paragraph in paragraphs)
    return f"{title}\n{chapter_body}\n"


def scrape() -> int:
    validate_configuration()

    current_url = STARTING_URL
    visited_urls: Set[str] = set()
    chapter_count = 0
    output_path = Path(OUTPUT_FILE)

    with requests.Session() as session:
        session.headers.update({"User-Agent": "WordpressScraper/1.0"})
        with output_path.open("w", encoding="utf-8") as output_handle:
            while True:
                if current_url in visited_urls:
                    raise RuntimeError(f"Detected a loop in the next-post chain at {current_url}.")

                visited_urls.add(current_url)
                LOGGER.info("Fetching %s", current_url)
                html = get_soup(current_url, session)
                output_handle.write(get_chapter(html, current_url))
                chapter_count += 1

                if current_url == ENDING_URL:
                    return chapter_count

                next_url = get_next_url(html)
                if not next_url:
                    raise RuntimeError(
                        "Could not find the next post before reaching ENDING_URL. "
                        f"Last successful URL: {current_url}"
                    )

                current_url = next_url


def main() -> None:
    configure_logging()

    try:
        chapter_count = scrape()
    except Exception as exc:  # noqa: BLE001
        LOGGER.error("Scrape failed: %s", exc)
        raise SystemExit(1) from exc

    LOGGER.info("Done. Wrote %s chapter(s) to %s.", chapter_count, OUTPUT_FILE)


if __name__ == "__main__":
    main()