release: 1.3.0 — split CI checks and automate PyPI publishing

.github/workflows/ci.yml

@@ -6,7 +6,55 @@ on:
   pull_request:
 
 jobs:
+  # Quality gates run once on the current Python — they don't vary by runtime.
+  # Each is its own job so it shows as a separate check on the PR.
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install ruff
+      - run: ruff check src/bytecrawl tests
+
+  format:
+    name: format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install ruff
+      - run: ruff format --check src/bytecrawl tests
+
+  typecheck:
+    name: type check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install (package + llm/mcp deps so mypy resolves imports)
+        run: pip install -e ".[llm,dev,mcp]"
+      - run: mypy src/bytecrawl
+
+  security:
+    name: security
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install bandit
+      - run: bandit -r src/bytecrawl -ll
+
   test:
+    name: unit test ${{ matrix.python-version }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,7 +71,5 @@ jobs:
       - name: Install MCP extra (needs Python 3.10+)
         if: matrix.python-version != '3.9'
         run: pip install -e ".[mcp]"
-      - name: Lint
-        run: ruff check src/bytecrawl tests
       - name: Tests
         run: pytest --cov=bytecrawl --cov-report=term-missing

.github/workflows/release.yml

@@ -14,6 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+    outputs:
+      # true only when this push introduced a new version (no tag yet) —
+      # the publish job below keys off it so PyPI never gets a duplicate.
+      publish: ${{ steps.tag.outputs.exists == 'false' }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -66,3 +70,25 @@ jobs:
           gh release create "v${{ steps.v.outputs.version }}" \
             --title "v${{ steps.v.outputs.version }}" \
             --notes-file release-notes.md
+
+  # Runs in the SAME workflow run right after `release`, only for a new version.
+  # (A separate workflow keyed on the tag/release would NOT fire: events made
+  # with GITHUB_TOKEN don't trigger further workflow runs.)
+  publish:
+    needs: release
+    if: needs.release.outputs.publish == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      # OIDC token for PyPI Trusted Publishing — no API token/secret needed.
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build sdist + wheel
+        run: |
+          pip install build
+          python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

CHANGELOG.md

@@ -1,4 +1,24 @@
 # Changelog
+## 1.3.0 — 2026-09-11
+
+### Changed
+- **CI is split into separate checks instead of one "test" job.** Lint
+  (`ruff check`), format (`ruff format --check`), type check (`mypy`) and
+  security (`bandit`) each run as their own top-level check alongside the
+  unit-test matrix (`unit test 3.9` / `unit test 3.12`). A red check now points
+  straight at the cause — a formatting slip no longer hides behind a failing
+  type check inside a single collapsed job.
+- **Releases publish to PyPI automatically** through GitHub Actions Trusted
+  Publishing (OIDC) — no API tokens or stored secrets. A push to `main` with a
+  new `__version__` runs the tests, tags the release, and uploads to PyPI;
+  nothing publishes without a version bump.
+
+### Internal
+- Version is single-sourced from `__version__`: a test now pins the top
+  CHANGELOG entry and both READMEs to it, so they can't drift.
+- Type-checking fixes across the scraping and crawl modules and one audited
+  `# nosec` on the MCP HTTP bind. No behaviour change.
+
 ## 1.2.2 — 2026-09-10
 
 ### Fixed

README.es.md

@@ -10,8 +10,8 @@ Shark-Search y OPIC en Python puro.
 - **Web**: https://bytecrawl.vercel.app/
 - **Endpoint MCP hosteado**: https://bytecrawl.vercel.app/mcp
 - **Skill para agentes**: https://bytecrawl.vercel.app/agent-onboarding/SKILL.md
-- **Última versión**: **1.2.2** — seis herramientas MCP, comparación de
-  estrategias, `Page.links()` absolutos ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md))
+- **Última versión**: **1.3.0** — publicación automática a PyPI y checks de CI
+  separados (lint, formato, tipos, seguridad) ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md))
 - **PyPI**: https://pypi.org/project/bytecrawl/
 - **GitHub**: https://github.com/abrahamperz/ByteCrawl
 

README.md

@@ -16,8 +16,8 @@ and OPIC in pure Python.
 - **Webpage**: https://bytecrawl.vercel.app/
 - **Hosted MCP endpoint**: https://bytecrawl.vercel.app/mcp
 - **Agent skill**: https://bytecrawl.vercel.app/agent-onboarding/SKILL.md
-- **Latest release**: **1.2.2** — six MCP tools, strategy comparison,
-  absolute `Page.links()` ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md))
+- **Latest release**: **1.3.0** — automated PyPI publishing and separated CI
+  gates (lint, format, types, security) ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md))
 
 ## Why focused crawling?
 

pyproject.toml

@@ -29,7 +29,7 @@ browser = ["playwright>=1.40"]
 llm = ["markdownify>=0.11", "trafilatura>=1.6"]
 mcp = ["mcp>=2; python_version >= '3.10'", "markdownify>=0.11", "trafilatura>=1.6"]
 all = ["playwright>=1.40", "markdownify>=0.11", "trafilatura>=1.6", "mcp>=2; python_version >= '3.10'"]
-dev = ["pytest>=7", "pytest-cov", "ruff"]
+dev = ["pytest>=7", "pytest-cov", "ruff", "mypy", "bandit"]
 
 [project.scripts]
 bytecrawl-mcp = "bytecrawl.mcp_server:main"
@@ -60,5 +60,14 @@ target-version = "py39"
 [tool.ruff.lint]
 select = ["E", "F", "W", "I", "UP", "B"]
 
+[tool.mypy]
+
+[[tool.mypy.overrides]]
+# playwright is the optional [browser] extra, not installed for the type-check
+# job — Scraper.browser() imports it lazily behind a try/except. Ignore its
+# missing stubs instead of pulling the heavy browser stack into CI.
+module = ["playwright.*"]
+ignore_missing_imports = true
+
 [tool.vercel]
 entrypoint = "landing.app:app"

src/bytecrawl/__init__.py

@@ -36,8 +36,17 @@
 )
 
 __all__ = [
-    "Scraper", "Page", "Session",
-    "Crawler", "BFS", "SharkSearch", "OPIC", "CrawlResult", "STRATEGIES",
-    "pagerank", "cosine", "compare",
+    "Scraper",
+    "Page",
+    "Session",
+    "Crawler",
+    "BFS",
+    "SharkSearch",
+    "OPIC",
+    "CrawlResult",
+    "STRATEGIES",
+    "pagerank",
+    "cosine",
+    "compare",
 ]
-__version__ = "1.2.2"
+__version__ = "1.3.0"

src/bytecrawl/crawl/__init__.py

@@ -27,6 +27,15 @@
 from .text import cosine, relevance
 
 __all__ = [
-    "Crawler", "BFS", "SharkSearch", "OPIC", "CrawlResult", "Frontier",
-    "STRATEGIES", "cosine", "relevance", "pagerank", "compare",
+    "Crawler",
+    "BFS",
+    "SharkSearch",
+    "OPIC",
+    "CrawlResult",
+    "Frontier",
+    "STRATEGIES",
+    "cosine",
+    "relevance",
+    "pagerank",
+    "compare",
 ]

src/bytecrawl/crawl/base.py

@@ -31,15 +31,17 @@ class Crawler:
 
     name = "bfs"
 
-    def __init__(self, query: str = "", delay: float = 0.2, timeout: int = 10,
-                 same_domain: bool = True):
+    def __init__(
+        self, query: str = "", delay: float = 0.2, timeout: int = 10, same_domain: bool = True
+    ):
         self.query = query
         self.same_domain = same_domain
         self.scraper = Scraper(delay=delay, timeout=timeout)
 
     # --- extension point -------------------------------------------------------
-    def score_links(self, url: str, links: list[dict], page_relevance: float,
-                    depth: int) -> list[tuple[str, float]]:
+    def score_links(
+        self, url: str, links: list[dict], page_relevance: float, depth: int
+    ) -> list[tuple[str, float]]:
         """BFS: the score only encodes depth (shallower = visited sooner).
 
         links: [{url, anchor}]. Returns [(url, score)] for the frontier.
@@ -89,19 +91,24 @@ def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlRe
 
             links = []
             for a in page.soup.select("a[href]"):
-                child = normalize(a["href"], url)
+                child = normalize(str(a["href"]), url)
                 if not child or child == url:
                     continue
                 if self.same_domain and _root_domain(urlparse(child).netloc) != domain:
                     continue
                 links.append({"url": child, "anchor": a.get_text(" ", strip=True)})
 
             result.graph[url] = [link["url"] for link in links]
-            result.pages.append({
-                "url": url, "title": title[:120], "score": round(score, 4),
-                "relevance": round(page_relevance, 4), "depth": depth,
-                "order": len(result.pages) + 1,
-            })
+            result.pages.append(
+                {
+                    "url": url,
+                    "title": title[:120],
+                    "score": round(score, 4),
+                    "relevance": round(page_relevance, 4),
+                    "depth": depth,
+                    "order": len(result.pages) + 1,
+                }
+            )
 
             self.on_visit(url, links)
 
@@ -117,8 +124,8 @@ def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlRe
             "elapsed": round(time.perf_counter() - t0, 2),
             "frontier_left": len(frontier),
             "relevant_found": len(result.relevant()) if self.query else None,
-            "avg_relevance": round(
-                sum(p["relevance"] for p in result.pages) / len(result.pages), 4
-            ) if result.pages else 0.0,
+            "avg_relevance": round(sum(p["relevance"] for p in result.pages) / len(result.pages), 4)
+            if result.pages
+            else 0.0,
         }
         return result

src/bytecrawl/crawl/compare.py

@@ -7,13 +7,15 @@
 from .strategies import STRATEGIES
 
 
-def _leg(name: str, start: str, query: str, max_pages: int, max_depth: int,
-         delay: float, timeout: int) -> dict:
+def _leg(
+    name: str, start: str, query: str, max_pages: int, max_depth: int, delay: float, timeout: int
+) -> dict:
     """One strategy's leg of a comparison. Never raises: a strategy that fails
     reports inside its own entry so the other two still come back."""
     try:
         result = STRATEGIES[name](query=query, delay=delay, timeout=timeout).crawl(
-            start, max_pages=max_pages, max_depth=max_depth)
+            start, max_pages=max_pages, max_depth=max_depth
+        )
     except Exception as e:
         return {"error": f"{type(e).__name__}: {e}"}
     return {
@@ -23,8 +25,14 @@ def _leg(name: str, start: str, query: str, max_pages: int, max_depth: int,
     }
 
 
-def compare(start: str, query: str, max_pages: int = 20, max_depth: int = 4,
-            delay: float = 0.2, timeout: int = 10) -> dict:
+def compare(
+    start: str,
+    query: str,
+    max_pages: int = 20,
+    max_depth: int = 4,
+    delay: float = 0.2,
+    timeout: int = 10,
+) -> dict:
     """Run every strategy over one site on the same budget, side by side.
 
     A single ordering looks like any other crawler's. The case for having
@@ -49,13 +57,16 @@ def compare(start: str, query: str, max_pages: int = 20, max_depth: int = 4,
     if not query:
         raise ValueError(
             "compare needs a query: with nothing to be relevant to, the "
-            "strategies aren't comparable (use a single crawler instead)")
+            "strategies aren't comparable (use a single crawler instead)"
+        )
     names = list(STRATEGIES)
     with ThreadPoolExecutor(max_workers=len(names)) as pool:
-        legs = list(pool.map(
-            lambda n: _leg(n, start, query, max_pages, max_depth,
-                           delay * len(names), timeout),
-            names))
+        legs = list(
+            pool.map(
+                lambda n: _leg(n, start, query, max_pages, max_depth, delay * len(names), timeout),
+                names,
+            )
+        )
     results = dict(zip(names, legs))
 
     ok = {n: r for n, r in results.items() if "error" not in r}

src/bytecrawl/crawl/graph.py

@@ -8,8 +8,9 @@
 from __future__ import annotations
 
 
-def pagerank(graph: dict[str, list[str]], damping: float = 0.85,
-             iterations: int = 30) -> dict[str, float]:
+def pagerank(
+    graph: dict[str, list[str]], damping: float = 0.85, iterations: int = 30
+) -> dict[str, float]:
     """PageRank via power iteration over the crawled graph.
 
     Offline version of the same concept OPIC approximates online: comparing

src/bytecrawl/crawl/result.py

@@ -10,7 +10,7 @@ class CrawlResult:
     """What a crawl returns: pages, graph and numbers for comparison."""
 
     strategy: str
-    pages: list[dict] = field(default_factory=list)   # url, title, score, relevance, depth, order
+    pages: list[dict] = field(default_factory=list)  # url, title, score, relevance, depth, order
     graph: dict[str, list[str]] = field(default_factory=dict)
     stats: dict = field(default_factory=dict)
 

src/bytecrawl/crawl/strategies.py

@@ -51,7 +51,7 @@ def score_links(self, url, links, page_relevance, depth):
         scored = []
         for link in links:
             url_words = " ".join(_tokens(urlparse(link["url"]).path))
-            local = relevance(f'{link["anchor"]} {url_words}', self.query)
+            local = relevance(f"{link['anchor']} {url_words}", self.query)
             score = self.gamma * inherited + (1 - self.gamma) * local
             self._inherited[link["url"]] = inherited
             scored.append((link["url"], score))
@@ -104,5 +104,7 @@ def score_links(self, url, links, page_relevance, depth):
 # strategy, so a crawl requested over HTTP, over MCP or in Python means the
 # same thing everywhere.
 STRATEGIES: dict[str, type[Crawler]] = {
-    "bfs": BFS, "shark": SharkSearch, "opic": OPIC,
+    "bfs": BFS,
+    "shark": SharkSearch,
+    "opic": OPIC,
 }

src/bytecrawl/crawl/text.py

@@ -38,7 +38,7 @@ def cosine(text_a: str, text_b: str) -> float:
 @lru_cache(maxsize=4096)
 def _trigrams(word: str) -> frozenset:
     padded = f"~{word}~"
-    return frozenset(padded[i:i + 3] for i in range(len(padded) - 2))
+    return frozenset(padded[i : i + 3] for i in range(len(padded) - 2))
 
 
 def _dice(a: frozenset, b: frozenset) -> float:
@@ -81,7 +81,7 @@ def _resolve_terms(query: str, vocabulary: set) -> str:
             score = _dice(grams, _trigrams(word))
             if score > best_score:
                 best, best_score = word, score
-        resolved.append(best if best_score >= _FUZZY_MIN else term)
+        resolved.append(best if best is not None and best_score >= _FUZZY_MIN else term)
     return " ".join(resolved)