python-hatch-workflow.yml

# This workflow installs Python dependencies and runs tests across versions.
#
# Cost design (to stay within GitHub Actions included minutes):
#   - Pull requests:  fast tests only, single Python version  (cheap, quick gate)
#   - Push to main:   fast (full matrix) + network + slow      (real integration gate)
#   - Nightly (cron): full suite                               (freshness vs SEC API)
#   - concurrency:    cancel superseded PR runs; main runs always complete
#   - pip cache:      avoid reinstalling deps from scratch in every job
#
# There is deliberately NO paths-ignore. A docs-only change would then report no
# checks at all, and a required status check that never reports blocks the pull
# request forever. Since these checks are meant to become required, every PR has
# to produce them — the few CI minutes a markdown-only run costs are cheaper than
# a merge queue that deadlocks on documentation.
#
# REQUIRED CHECKS ARE PINNED TO JOB NAMES. `main` requires exactly two contexts:
#
#     Cassette safety gate
#     test-fast (3.13)
#
# The second name is generated from the matrix below, which is event-dependent —
# a pull request builds only 3.13, so the context is literally "test-fast (3.13)".
# Renaming a job, or changing the pull-request leg of that matrix, retires the
# context that branch protection is waiting for; it never reports, and every pull
# request blocks forever with enforce_admins on. If you touch either, update the
# protection contexts in the same commit:
#
#     gh api repos/dgunning/edgartools/branches/main/protection \
#        --jq .required_status_checks.contexts
#
# test-network, test-slow and Combine Coverage are
# deliberately NOT required: they carry `if: github.event_name != 'pull_request'`
# and so report as skipped on every pull request. They run on push to main,
# which is the integration gate.
#
# Because they cannot gate, they are REPORTED instead — see the `report` job at
# the end of this file. Combine Coverage is where the 65% floor is enforced
# (`coverage report --fail-under=65`), and until 2026-08-10 a failure there, or
# in test-network or test-slow, appeared in the Actions tab and nowhere else.
# That is the same hole #999 closed for the regression workflow after a red main
# survived three merges unnoticed (bead edgartools-hwdp); this workflow was left
# out of that fix even though it carries more.

name: Build and Test Edgartools

# 5.*.x: the short-lived maintenance branch cut from the last 5.x tag during the
# 6.0 freeze window (bead edgartools-07lk.22). Without it a backport PR into that
# branch would report no checks at all. Branch protection is not configured on it,
# so these checks inform but do not block.
#
# Editing THIS copy does not change what such a branch runs. A push runs the
# workflow file at the pushed commit, and a pull request runs it from the merge
# commit of head into base, so a branch cut from a 5.x tag runs the copy in that
# tag, which still tests Python 3.10. In main's copy the 5.*.x entries are
# therefore inert; they stay because removing them buys nothing.
on:
  push:
    branches: [ "main", "5.*.x" ]
  pull_request:
    branches: [ "main", "5.*.x" ]
  schedule:
    # Nightly full suite at 07:00 UTC
    - cron: '0 7 * * *'
  workflow_dispatch:

# Cancel an in-progress run when a newer commit lands on the same PR. Pushes to
# main are never cancelled: PRs merge without being up to date with main, so the
# post-merge run is the only check that the combination works, and back-to-back
# merges used to cancel it before it finished.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name != 'push' }}

jobs:
  # Must pass before any job that runs pytest. vcrpy loads cassettes with
  # PyYAML's unsafe loader, so a cassette carrying a python/ tag executes code
  # in the runner the moment a test touches it — gating after the suite has
  # started would be no gate at all. Cheap (~10s) and needs only PyYAML.
  cassette-gate:
    name: Cassette safety gate
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python 3.13
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: '3.13'
    - name: Install PyYAML
      run: python -m pip install --upgrade pip PyYAML
    - name: Scan cassettes for unsafe YAML tags
      run: python scripts/check_cassettes.py

  test-fast:
    # PRs to main: single version for a quick gate. Push/schedule: the declared
    # floor (requires-python >=3.11) plus 3.13, so the minimum is exercised rather
    # than merely asserted. A 5.*.x maintenance branch tests 3.10 from its own
    # copy of this file (see the note above `on:`), not from this one.
    # The main-PR leg must stay exactly ["3.13"]: branch protection requires the
    # context "test-fast (3.13)" (see the header of this file).
    needs: cassette-gate
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ (github.event_name == 'pull_request' && github.base_ref == 'main') && fromJSON('["3.13"]') || fromJSON('["3.11", "3.13"]') }}

    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e ".[ai]"
        python -m pip install pytest pytest-cov pytest-env pytest-xdist pytest-asyncio pytest-retry pytest-mock pytest-vcr "vcrpy>=8.3.0,<9" freezegun==1.5.1 filelock tqdm responses

    # The floor leg also holds the pandas floor's major. We support pandas 2.x
    # and 3.x, and until the Python floor rose pandas 2 was tested only by
    # accident: pandas 3 requires Python 3.11, so the old 3.10 leg resolved to
    # pandas 2.3.x while 3.13 got 3.x. Pinning here keeps both majors under test.
    - name: Pin pandas 2.x on the floor leg
      if: matrix.python-version == '3.11'
      run: python -m pip install "pandas>=2.0.0,<3"

    # A source scan, so it runs here rather than as its own job: this is the
    # only context pull requests are required to report, and adding a second
    # required one means updating branch protection in lockstep (see the header
    # of this file). It is also why the check reads source instead of hooking
    # pytest — half the regression tree is network-marked and runs post-merge,
    # so a runtime hook would not see those tests on a pull request at all.
    - name: Check no regression test skips itself
      run: python scripts/check_regression_skips.py

    # Same reasoning, different property: a regression test has to name the bug
    # it guards, or nobody can answer "is this still reachable?" and the tree can
    # never be pruned. Static like the check above, and for the same reason — it
    # has to see the post-merge half of the tree on a pull request.
    - name: Check every regression test names its bug
      run: python scripts/check_regression_provenance.py

    # The one check here that has to RUN the tests rather than read them: no
    # source scan can tell whether a test reaches the SEC, because the answer
    # depends on what the restored HTTP cache happens to be holding. Sockets
    # are blocked instead, which gives the same answer twice.
    #
    # Scoped to the branch's own test files on a pull request (seconds) and run
    # over the whole lane on push (~7 min), because a test can only become
    # network-dependent in a file the branch touched, and main is where a
    # source-side cause — a deleted cassette, say — gets caught. Not its own
    # job, for the branch-protection reason above.
    - name: Check no new test needs the SEC while marked fast
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        set -euo pipefail
        if [ "${{ github.event_name }}" = "pull_request" ]; then
          # The API, not `git diff`: this checkout is shallow, so no merge base
          # with the target branch exists to diff against. Only test modules
          # (pytest's test_*.py / *_test.py): a helper under tests/ collects
          # nothing, and the audit reads an empty run as a broken one (#1527).
          CHANGED=$(gh api --paginate \
            "repos/${{ github.repository }}/pulls/${{ github.event.number }}/files" \
            --jq '.[] | select(.status != "removed") | .filename' \
            | grep -E '^tests/(.*/)?(test_[^/]*|[^/]*_test)\.py$' | tr '\n' ' ' || true)
          if [ -z "$CHANGED" ]; then
            echo "No test files changed; nothing to audit."
            exit 0
          fi
          echo "Auditing changed test files: $CHANGED"
          python scripts/check_offline_audit.py $CHANGED
        else
          python scripts/check_offline_audit.py
        fi

    # A pull request that stages a 6.0 change (a deprecation, a will-raise
    # warning, "removed in 6.0") has to touch docs/upgrade/6.0.md in the same PR.
    # #1200/#1201 shipped 5.55.0 deprecations with no guide entry, and nothing
    # mechanical noticed (bead edgartools-07lk.14). Pull requests only: the check
    # reads the PR's diff, and main has no PR to read. Labels are read live
    # rather than from the event payload, so after adding `no-upgrade-guide`,
    # re-run this job instead of pushing an empty commit.
    - name: Check a staged 6.0 change updates the upgrade guide
      if: github.event_name == 'pull_request'
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        set -euo pipefail
        gh api --paginate \
          "repos/${{ github.repository }}/pulls/${{ github.event.number }}/files" \
          --jq '.[] | {filename, status, patch} | tojson' > "$RUNNER_TEMP/pr_files.jsonl"
        PR_LABELS=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.number }}/labels" \
          --jq '[.[].name] | join(",")') \
          python scripts/check_upgrade_guide.py "$RUNNER_TEMP/pr_files.jsonl"

    - name: Run fast tests
      run: |
        # Parallelize fast tests - safe, no SEC API calls.
        #
        # No `and not regression` here, unlike the network and slow jobs. The
        # regression tree is 1,746 tests and every one of them used to be
        # deselected on pull requests, so a PR could break any of them and merge
        # green (bead edgartools-07lk.21). conftest.py now classifies each as
        # fast or network by measurement, and the 934 offline ones gate here.
        # The network half still runs only in the Regression Tests workflow —
        # PRs must not hammer the SEC endpoint.
        pytest -n auto --cov --cov-report=xml -m 'fast'

    - name: Upload coverage data
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      if: matrix.python-version == '3.13' && github.event_name != 'pull_request'
      with:
        name: coverage-fast
        path: |
          .coverage
          coverage.xml
        include-hidden-files: true
        retention-days: 1

  test-network:
    # Network tests skip PRs — SEC API behavior doesn't change per-commit.
    # Runs on push to main, nightly schedule, and manual dispatch.
    if: github.event_name != 'pull_request'
    needs: cassette-gate
    runs-on: ubuntu-latest
    env:
      # Relocate the EDGAR data/cache dir into the workspace so it can be cached.
      # This only moves where the HTTP cache (_tcache) lives; it does NOT enable
      # local storage (that needs EDGAR_USE_LOCAL_DATA, which tests disable).
      EDGAR_LOCAL_DATA_DIR: ${{ github.workspace }}/.edgar-data

    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python 3.12
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "3.12"
        cache: 'pip'

    # Persist the SEC HTTP cache across runs. Filing documents and XBRL under
    # /Archives/edgar/data are cached forever (see CACHE_RULES), so a warm cache
    # serves the bulk of the suite from disk instead of re-downloading from SEC.
    # run_id-suffixed key always writes a fresh entry; restore-keys warm-starts
    # from the most recent prior cache (monotonic, GH evicts LRU at the 10GB cap).
    - name: Restore SEC HTTP cache
      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      with:
        path: ${{ github.workspace }}/.edgar-data/_tcache
        key: edgar-tcache-network-${{ github.run_id }}
        restore-keys: |
          edgar-tcache-network-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e ".[ai]"
        python -m pip install pytest pytest-cov pytest-env pytest-xdist pytest-asyncio pytest-retry pytest-mock pytest-vcr "vcrpy>=8.3.0,<9" freezegun==1.5.1 filelock tqdm responses

    - name: Run network tests
      run: |
        # Sequential - respects SEC rate limits. --enable-cache persists HTTP
        # responses to _tcache (immutable /Archives data cached forever).
        pytest --cov --cov-report=xml -m 'network and not slow and not regression' --enable-cache

    - name: Upload coverage data
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: coverage-network
        path: |
          .coverage
          coverage.xml
        include-hidden-files: true
        retention-days: 1

  test-slow:
    # Slow tests skip PRs (often network-heavy). Push/schedule/dispatch only.
    if: github.event_name != 'pull_request'
    needs: cassette-gate
    runs-on: ubuntu-latest
    env:
      EDGAR_LOCAL_DATA_DIR: ${{ github.workspace }}/.edgar-data

    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python 3.12
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "3.12"
        cache: 'pip'

    # Persist the SEC HTTP cache (see test-network for rationale).
    - name: Restore SEC HTTP cache
      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      with:
        path: ${{ github.workspace }}/.edgar-data/_tcache
        key: edgar-tcache-slow-${{ github.run_id }}
        restore-keys: |
          edgar-tcache-slow-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e ".[ai]"
        python -m pip install pytest pytest-cov pytest-env pytest-xdist pytest-asyncio pytest-retry pytest-mock pytest-vcr "vcrpy>=8.3.0,<9" freezegun==1.5.1 filelock tqdm responses

    - name: Run slow tests
      run: |
        # Sequential - often network-heavy. --enable-cache persists HTTP responses.
        pytest --cov --cov-report=xml -m 'slow and not regression' --enable-cache

    - name: Upload coverage data
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: coverage-slow
        path: |
          .coverage
          coverage.xml
        include-hidden-files: true
        retention-days: 1

  coverage:
    name: Combine Coverage
    # Only meaningful when the full suite ran (not on PRs).
    if: github.event_name != 'pull_request'
    needs: [test-fast, test-network, test-slow]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "3.11"
        cache: 'pip'
    - name: Install coverage
      run: pip install coverage[toml]

    - name: Download coverage data (fast)
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: coverage-fast
        path: coverage-fast/

    - name: Download coverage data (network)
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: coverage-network
        path: coverage-network/

    - name: Download coverage data (slow)
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: coverage-slow
        path: coverage-slow/

    - name: Combine coverage
      run: |
        # Move .coverage files with unique names
        mv coverage-fast/.coverage .coverage.fast
        mv coverage-network/.coverage .coverage.network
        mv coverage-slow/.coverage .coverage.slow

        # Combine all coverage data
        coverage combine

        # Generate report and check threshold
        coverage report --fail-under=65
        coverage xml -o combined-coverage.xml

    - name: Upload combined coverage to Codecov
      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
      with:
        files: combined-coverage.xml
        fail_ci_if_error: false
        token: ${{ secrets.CODECOV_TOKEN }}

  # A red `main` has to arrive somewhere a person looks.
  #
  # Ported from regression-tests.yml, which got this treatment in #999 after the
  # post-merge regression lane went red on 2026-08-08 and three more pull
  # requests merged through it before anyone noticed. THIS workflow never got
  # one, and it is the lane that carries more: the 65% coverage floor
  # (`coverage report --fail-under=65`), plus test-network, test-slow and
  # (test-strict-errors ran here too until 6.0 made its flag the only behaviour). All of them are post-merge only, none is a required check, and until now none of
  # them could tell anybody they had failed.
  #
  # Note what this deliberately does NOT do: make Combine Coverage required.
  # It cannot be. It `needs` test-network and test-slow, which are skipped on
  # pull requests, so a required Combine Coverage would never report and would
  # block every pull request forever — the deadlock described at the top of this
  # file. Reporting is the right mechanism for a check that cannot gate.
  #
  # `needs` names every post-merge job on purpose. A failure in test-network
  # SKIPS Combine Coverage rather than failing it, so keying on the coverage
  # result alone would stay silent for the failure that stops the threshold
  # being evaluated at all.
  #
  # Its own label and title, separate from regression-tests.yml's `red-main`.
  # Sharing one issue would have the two lanes fight over it — whichever went
  # green last would close an issue the other still needed open.
  report:
    name: Report red main (build)
    needs: [cassette-gate, test-fast, test-network, test-slow, coverage]
    # main only. A push to a 5.*.x maintenance branch also runs this workflow, and
    # a green one would otherwise CLOSE main's open red-main issue.
    if: always() && github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    permissions:
      issues: write
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      REPO: ${{ github.repository }}
      RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
      SHA: ${{ github.sha }}
      LABEL: red-main-build
      TITLE: "Build and Test Edgartools is failing on main"
      R_GATE: ${{ needs.cassette-gate.result }}
      R_FAST: ${{ needs.test-fast.result }}
      R_NETWORK: ${{ needs.test-network.result }}
      R_SLOW: ${{ needs.test-slow.result }}
      R_COVERAGE: ${{ needs.coverage.result }}
    steps:
    - name: Ensure the tracking label exists
      run: |
        gh label create "$LABEL" --repo "$REPO" --color b60205 \
          --description "Post-merge build/coverage lane is red" 2>/dev/null || true

    - name: Decide whether the lane is red
      id: verdict
      run: |
        red=0
        for r in "$R_GATE" "$R_FAST" "$R_NETWORK" "$R_SLOW" "$R_COVERAGE"; do
          [ "$r" = "failure" ] && red=1
        done
        echo "red=$red" >> "$GITHUB_OUTPUT"

    - name: Open or update the tracking issue
      if: steps.verdict.outputs.red == '1'
      run: |
        number=$(gh issue list --repo "$REPO" --label "$LABEL" --state all \
                   --limit 1 --json number --jq '.[0].number // empty')
        # Name the job that failed. This lane has five, and "the build is red"
        # without saying which one sends the reader to the Actions tab to find
        # out — which is the step nobody takes.
        body=$(printf '%s\n\n- commit: %s\n- run: %s\n\n| job | result |\n| --- | --- |\n| Cassette safety gate | %s |\n| test-fast | %s |\n| test-network | %s |\n| test-slow | %s |\n| Combine Coverage (65%% floor) | %s |\n' \
                 "The post-merge build lane failed on \`main\`." "$SHA" "$RUN_URL" \
                 "$R_GATE" "$R_FAST" "$R_NETWORK" "$R_SLOW" "$R_COVERAGE")
        if [ -z "$number" ]; then
          gh issue create --repo "$REPO" --label "$LABEL" \
            --title "$TITLE" --body "$body"
        else
          gh issue reopen --repo "$REPO" "$number" 2>/dev/null || true
          gh issue comment --repo "$REPO" "$number" --body "$body"
        fi

    - name: Close the tracking issue when main is green again
      if: steps.verdict.outputs.red == '0'
      run: |
        number=$(gh issue list --repo "$REPO" --label "$LABEL" --state open \
                   --limit 1 --json number --jq '.[0].number // empty')
        if [ -n "$number" ]; then
          gh issue close --repo "$REPO" "$number" \
            --comment "Green again at $SHA — $RUN_URL"
        fi