WebScraper-GitHub-User.py

#!/usr/bin/env python3
# https://github.com/VictorGabriel7Dev/webscraper-github-user
# https://victorgabriel.dev/projetos/webscraper-github-user
# SPDX-License-Identifier: AGPL-3.0-or-later
# Version: 1.01
#
# Author: Victor Gabriel
# Site: https://victorgabriel.dev
# E-mail: contato@victorgabriel.dev
# GitHub: https://github.com/VictorGabriel7Dev
# LinkedIn: https://www.linkedin.com/in/victorgabriel-dev
# WhatsApp: https://wa.me/@VictorGabriel_Dev
# Discord: @VictorGabriel.dev https://discord.com/users/1481407654458036265
# Telegram: https://t.me/VictorGabriel_Dev
# Instagram: https://www.instagram.com/VictorGabriel_Dev
# Generated by: Claude https://claude.ai

"""
webscraper-github-user
====================
A python script for Web Scrapping GitHub Users.
Scrapes a GitHub user's starred repositories and following list
directly from the GitHub HTML pages, no API token required.

Usage:
    python WebScraper-GitHub-User.py <github_username>

Example:
    python WebScraper-GitHub-User.py torvalds
"""

import sys
import re
import html
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError


# ── ANSI colour palette ──────────────────────────────────────────────────────

class Colour:
    RESET   = "\033[0m"
    BOLD    = "\033[1m"
    DIM     = "\033[2m"
    CYAN    = "\033[96m"
    GREEN   = "\033[92m"
    YELLOW  = "\033[93m"
    MAGENTA = "\033[95m"
    RED     = "\033[91m"
    WHITE   = "\033[97m"
    GREY    = "\033[90m"


# ── Helpers ──────────────────────────────────────────────────────────────────

HEADERS = {
    "Host": "github.com",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
}


def fetch(url: str) -> str:
    """Fetch a URL and return the response body as a string."""
    req = Request(url, headers=HEADERS)
    try:
        with urlopen(req, timeout=20) as resp:
            return resp.read().decode("utf-8", errors="replace")
    except HTTPError as exc:
        print(
            f"{Colour.RED}✗  HTTP {exc.code} while fetching:{Colour.RESET} {url}",
            file=sys.stderr
        )
        sys.exit(1)
    except URLError as exc:
        print(
            f"{Colour.RED}✗  Connection error:{Colour.RESET} {exc.reason}",
            file=sys.stderr
        )
        sys.exit(1)


def banner(username: str) -> None:
    w = 60
    print()
    print(f"{Colour.CYAN}{Colour.BOLD}{'─' * w}{Colour.RESET}")
    print(
        f"{Colour.CYAN}{Colour.BOLD}"
        f"  ⬡  webscraper-github-user"
        f"{Colour.RESET}"
    )
    print(
        f"{Colour.GREY}  Target → {Colour.WHITE}{Colour.BOLD}"
        f"github.com/{username}"
        f"{Colour.RESET}"
    )
    print(f"{Colour.CYAN}{Colour.BOLD}{'─' * w}{Colour.RESET}")
    print()


def section_header(title: str) -> None:
    print(f"\n{Colour.YELLOW}{Colour.BOLD}  ★  {title}{Colour.RESET}")
    print(f"{Colour.GREY}  {'·' * 50}{Colour.RESET}")


def summary(label: str, count: int) -> None:
    print(
        f"\n{Colour.GREEN}  ✔  {label}: "
        f"{Colour.BOLD}{count}{Colour.RESET}"
    )


# ── Stars scraper ─────────────────────────────────────────────────────────────

def scrape_stars(username: str) -> int:
    """Yield every starred repo for *username*. Returns total count."""
    section_header(f"Starred repositories: {username}")

    url: str = (
        f"https://github.com/{username}"
        f"?direction=desc&sort=stars&tab=stars"
    )
    visited_pages: set[str] = set()
    total = 0

    while url:
        if url in visited_pages:
            print(
                f"{Colour.GREY}  ↩  Duplicate page detected, stopping.{Colour.RESET}"
            )
            break
        visited_pages.add(url)

        body = fetch(url)
        lines = body.splitlines()

        repo = ""
        description_parts: list[str] = []
        collecting_desc = False
        next_url = ""

        i = 0
        while i < len(lines):
            line = lines[i]

            # Repository anchor  →  <a href="/user/repo">
            m = re.search(r'<\s*a\s+href=["\']/([\w.\-]+/[\w.\-]+)["\']\s*>\s*$', line, re.I)
            if m:
                repo = m.group(1)
                description_parts = []
                collecting_desc = False

            # Start of description paragraph
            if re.search(
                r'<\s*p\s+[^>]*(?:itemprop=["\']description["\']|'
                r'class=["\'][^"\']*(?:d-inline-block|col-9)[^"\']*["\'])[^>]*>',
                line, re.I
            ):
                collecting_desc = True
                # grab inline text on the same line
                inline = re.sub(r"<[^>]+>", "", line).strip()
                if inline:
                    description_parts.append(inline)

            elif collecting_desc:
                if re.search(r"<\s*/\s*p\s*>", line, re.I):
                    # end of description
                    raw = " ".join(description_parts)
                    desc = html.unescape(re.sub(r"<[^>]+>", "", raw)).strip(" .")
                    _print_star(total + 1, repo, desc or "<no description>")
                    total += 1
                    repo = ""
                    description_parts = []
                    collecting_desc = False
                else:
                    text = re.sub(r"<[^>]+>", "", line).strip()
                    if text:
                        description_parts.append(text)

            # Flush repo with empty description when metadata div appears
            if (
                repo
                and not collecting_desc
                and re.search(
                    r'<\s*div\s+[^>]*class=["\'][^"\']*'
                    r'(?:f6|color-fg-muted|mt-2)[^"\']*["\']',
                    line, re.I
                )
            ):
                _print_star(total + 1, repo, "<no description>")
                total += 1
                repo = ""

            # "Next" pagination link
            m_next = re.search(
                r'href=["\'](https://github\.com/' + re.escape(username) +
                r'\?after=[^"\']+(?:&(?:amp;)?|&amp;)tab=stars[^"\']*)["\']'
                r'[^>]*>\s*Next\s*<',
                line, re.I
            )
            if m_next:
                next_url = html.unescape(m_next.group(1))

            i += 1

        url = next_url  # empty string → loop ends

    summary("Total starred repositories", total)
    return total


def _print_star(index: int, repo: str, desc: str) -> None:
    idx_str = f"{Colour.GREY}{index:>4}.{Colour.RESET}"
    repo_str = f"{Colour.CYAN}{Colour.BOLD}{repo}{Colour.RESET}"
    desc_str = f"{Colour.DIM}{desc}{Colour.RESET}"
    print(f"  {idx_str}  {repo_str}")
    print(f"        {desc_str}")


# ── Following scraper ─────────────────────────────────────────────────────────

def scrape_following(username: str) -> int:
    """Print every account that *username* follows. Returns total count."""
    section_header(f"Following: {username}")

    url: str = f"https://github.com/{username}?tab=following"
    visited_pages: set[str] = set()
    seen_users: set[str] = set()
    total = 0

    while url:
        if url in visited_pages:
            print(
                f"{Colour.GREY}  ↩  Duplicate page detected, stopping.{Colour.RESET}"
            )
            break
        visited_pages.add(url)

        body = fetch(url)
        next_url = ""

        for line in body.splitlines():
            # Extract followed usernames from hovercard links
            m = re.search(
                r'["\']\/(?:orgs|users)\/([\w.\-]+)\/hovercard["\']',
                line, re.I
            )
            if m:
                user = m.group(1)
                if user not in seen_users:
                    seen_users.add(user)
                    total += 1
                    _print_following(total, user)

            # "Next" pagination link
            m_next = re.search(
                r'href=["\'](https://github\.com/' + re.escape(username) +
                r'\?page=(\d+)&(?:amp;)?tab=following)["\']'
                r'[^>]*>\s*Next\s*<',
                line, re.I
            )
            if m_next:
                candidate_url = html.unescape(m_next.group(1))
                # Make sure "Next" is not disabled
                disabled = re.search(
                    r'class=["\'][^"\']*(?:disabled|color-fg-muted)[^"\']*["\']'
                    r'[^>]*>\s*Next\s*<',
                    line, re.I
                )
                if not disabled:
                    next_url = candidate_url

        url = next_url

    summary("Total following", total)
    return total


def _print_following(index: int, user: str) -> None:
    idx_str = f"{Colour.GREY}{index:>4}.{Colour.RESET}"
    user_str = f"{Colour.MAGENTA}{Colour.BOLD}{user}{Colour.RESET}"
    url_str  = f"{Colour.DIM}https://github.com/{user}{Colour.RESET}"
    print(f"  {idx_str}  {user_str}  {url_str}")


# ── Entry point ───────────────────────────────────────────────────────────────

def main() -> None:
    if len(sys.argv) < 2:
        print(
            f"\n{Colour.RED}Usage:{Colour.RESET}  "
            f"python WebScraper-GitHub-User.py <github_username>\n",
            file=sys.stderr
        )
        sys.exit(1)

    username = sys.argv[1].strip().lstrip("@")
    banner(username)

    stars_count     = scrape_stars(username)
    following_count = scrape_following(username)

    # ── Final summary ────────────────────────────────────────────────────────
    w = 60
    print(f"\n{Colour.CYAN}{Colour.BOLD}{'─' * w}{Colour.RESET}")
    print(f"{Colour.BOLD}  Summary for {Colour.CYAN}{username}{Colour.RESET}")
    print(
        f"  {Colour.YELLOW}★{Colour.RESET}  Stars     : "
        f"{Colour.BOLD}{stars_count}{Colour.RESET}"
    )
    print(
        f"  {Colour.MAGENTA}●{Colour.RESET}  Following : "
        f"{Colour.BOLD}{following_count}{Colour.RESET}"
    )
    print(f"{Colour.CYAN}{Colour.BOLD}{'─' * w}{Colour.RESET}\n")


if __name__ == "__main__":
    main()