From 75ac428187b24f3b911336465fd124583cb6e5b2 Mon Sep 17 00:00:00 2001 From: Stanislas Poisson Date: Mon, 26 Oct 2026 23:53:25 +0100 Subject: [PATCH] feat(release): #125 publish the dataset from the kind of change --- .github/scripts/compare_exports.py | 57 +++++++- .github/workflows/build-dataset.yml | 203 +++++++++++++++++++++------- .github/workflows/watch-sources.yml | 43 ++++-- README.md | 2 +- docs/release.md | 30 ++-- 5 files changed, 261 insertions(+), 74 deletions(-) diff --git a/.github/scripts/compare_exports.py b/.github/scripts/compare_exports.py index 6288fdc..ce6b0bb 100755 --- a/.github/scripts/compare_exports.py +++ b/.github/scripts/compare_exports.py @@ -4,8 +4,15 @@ Usage: compare_exports.py OLD_DIR NEW_DIR [SUMMARY_FILE] It prints a summary in Markdown (rows added, rows removed, rows modified and the columns that changed, per file), -writes it to SUMMARY_FILE when given, and sets `changed` (true or false) in GITHUB_OUTPUT. Every column counts: -the points of the BAN are part of the dataset. +writes it to SUMMARY_FILE when given, and sets in GITHUB_OUTPUT: + +- `changed` (true or false): every column counts, the points of the BAN are part of the dataset; +- `kind`: `none`, `patch`, `minor` or `major`, from what changed (see docs/release.md): + - `patch`: only the points of the BAN moved (`address_count`, `latitude`, `longitude` of cities.csv); + - `minor`: anything else in the rows (a row added or removed, another column modified); + - `major`: the structure changed (the columns of a file, a file added or removed); +- `review` (true or false) and `review_reason`: a major change, or more than REMOVAL_THRESHOLD percent (1 by default) of + the rows of a file removed, must be checked by the maintainer before it is published. Rows are matched by their `id` column. A file without `id` is compared as a set of rows. """ @@ -16,6 +23,11 @@ from pathlib import Path +# The columns that move with the points of the BAN, without any file of INSEE or La Poste changing. +BAN_COLUMNS = {'cities.csv': {'address_count', 'latitude', 'longitude'}} +DEFAULT_REMOVAL_THRESHOLD = 1.0 + + def read(path): with open(path, newline='', encoding='utf-8') as handle: reader = csv.reader(handle) @@ -24,7 +36,7 @@ def read(path): def compare(old_path, new_path): - """Returns (added, removed, modified, columns, header_changed) for one file.""" + """Returns (added, removed, modified, columns, header_changed, old_rows) for one file.""" old_header, old_rows = read(old_path) new_header, new_rows = read(new_path) header_changed = old_header != new_header @@ -42,40 +54,73 @@ def compare(old_path, new_path): if different: modified += 1 columns.update(different) - return added, removed, modified, columns, header_changed + return added, removed, modified, columns, header_changed, len(old_rows) old_set = Counter(map(tuple, old_rows)) new_set = Counter(map(tuple, new_rows)) - return sum((new_set - old_set).values()), sum((old_set - new_set).values()), 0, Counter(), header_changed + return sum((new_set - old_set).values()), sum((old_set - new_set).values()), 0, Counter(), header_changed, len(old_rows) + + +def classify(name, added, removed, modified, columns, header_changed): + """The kind of change of one file: none, patch, minor or major.""" + if header_changed: + return 'major' + if added or removed: + return 'minor' + if modified: + return 'patch' if set(columns) <= BAN_COLUMNS.get(name, set()) else 'minor' + return 'none' def main(): old_dir, new_dir = Path(sys.argv[1]), Path(sys.argv[2]) names = sorted({p.name for p in old_dir.glob('*.csv')} | {p.name for p in new_dir.glob('*.csv')}) + threshold = float(os.environ.get('REMOVAL_THRESHOLD') or DEFAULT_REMOVAL_THRESHOLD) + order = ['none', 'patch', 'minor', 'major'] lines = ['| File | Added | Removed | Modified | Columns modified |', '| ---- | ----: | ------: | -------: | ---------------- |'] changed = False + kind = 'none' + reasons = [] for name in names: old, new = old_dir / name, new_dir / name if not old.exists() or not new.exists(): lines.append(f"| {name} | {'file added' if not old.exists() else ''} | {'file removed' if not new.exists() else ''} | | |") changed = True + kind = 'major' + reasons.append(f"{name}: file {'added' if not old.exists() else 'removed'}") continue - added, removed, modified, columns, header_changed = compare(old, new) + added, removed, modified, columns, header_changed, old_count = compare(old, new) if added or removed or modified or header_changed: changed = True + file_kind = classify(name, added, removed, modified, columns, header_changed) + kind = max(kind, file_kind, key=order.index) + if header_changed: + reasons.append(f'{name}: the columns changed') + if not header_changed and old_count and removed * 100 / old_count > threshold: + reasons.append(f'{name}: {removed} of {old_count} rows removed (more than {threshold:g} percent)') detail = ', '.join(f'{column} ({count})' for column, count in sorted(columns.items())) if header_changed: detail = 'the columns of the file changed' lines.append(f'| {name} | {added} | {removed} | {modified} | {detail} |') + review = kind == 'major' or bool(reasons) + if kind == 'major' and not reasons: + reasons.append('the structure of the files changed') + summary = '\n'.join(lines) + summary += f"\n\nKind of change: **{kind}**." + if reasons: + summary += ' To review: ' + '; '.join(reasons) + '.' print(summary) if len(sys.argv) > 3: Path(sys.argv[3]).write_text(summary + '\n', encoding='utf-8') if os.environ.get('GITHUB_OUTPUT'): with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as out: out.write(f"changed={'true' if changed else 'false'}\n") + out.write(f'kind={kind}\n') + out.write(f"review={'true' if review else 'false'}\n") + out.write(f"review_reason={'; '.join(reasons)}\n") if __name__ == '__main__': diff --git a/.github/workflows/build-dataset.yml b/.github/workflows/build-dataset.yml index e2176ca..85badce 100644 --- a/.github/workflows/build-dataset.yml +++ b/.github/workflows/build-dataset.yml @@ -1,24 +1,36 @@ name: Build dataset -# Builds a new version of the dataset and proposes it as a draft release (see docs/release.md). +# Builds a new version of the dataset and publishes it, or proposes it as a draft release to review (see docs/release.md). # # The database of the update holds the history and the identifiers of the dataset, so the build starts from the # SQL of the baseline release (the latest one by default), restored in a fresh database. It stops if the schema of the restored tables is not # the one of the migrations, if an identifier changed or disappeared, if the update is incomplete, or if a file does not # respect its schema. # -# The draft is created only when the export differs from the baseline release, in any column: a newer source file -# does not mean that the data changed. Otherwise the issue gets a comment and a marker, so that `Watch sources` -# does not start the build again for the same sources. +# The export is compared with the baseline release, in every column (a newer source file does not mean that the data +# changed), and the kind of change gives the version: # -# The maintainer reviews the draft and publishes it: that creates the tag and starts `Publish on data.gouv.fr`. -# Started by `Watch sources` when a source is newer, or by hand. With `dry_run` on, no draft and no comment. +# - nothing differs: no release, the issue gets a comment and a marker, so that `Watch sources` does not start the +# build again for the same sources; +# - patch (only the points of the BAN of cities.csv): published when `publish_patch` is on (Monday, or `force`), +# otherwise kept for that day, with a marker; +# - minor (any other change of the rows): published at once; +# - major (the structure of the files changed), or more than 1 percent of the rows of a file removed: a draft release, +# and a notification, because the maintainer has to review it. +# +# Publishing creates the signed tag on main and the release with the token RELEASE_TOKEN (a maintainer: the rules of +# the tags only let one create them), which starts `Publish on data.gouv.fr` and `Stamp schemas`. Without that token the +# release stays a draft. Started by `Watch sources` or by hand. With `dry_run` on, nothing is created and nothing is sent. on: workflow_dispatch: inputs: version: - description: 'The version of the release (default: the next minor version when a source of INSEE or La Poste is newer, 4.0.0 gives 4.1.0, otherwise the next patch version, 4.0.0 gives 4.0.1)' + description: 'The version of the release, always proposed as a draft (default: from the kind of change, 4.0.0 gives 4.1.0 for a minor change, 4.0.1 for a patch one, 5.0.0 for a major one)' required: false + publish_patch: + description: 'Publish a patch release (only the points of the BAN changed): set by Watch sources on Monday' + type: boolean + default: false baseline: description: 'The release whose SQL the build starts from and is compared with (default: the latest release)' required: false @@ -41,6 +53,7 @@ concurrency: env: GH_TOKEN: ${{ github.token }} COMPOSE: docker compose -p frenchpostalcode + NTFY_URL: ${{ secrets.NTFY_URL }} jobs: build: @@ -48,7 +61,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 300 steps: + # The token of the maintainer pushes the tag of a release (see the rules of the tags); without it, no release is published. - uses: actions/checkout@v7 + with: + token: ${{ secrets.RELEASE_TOKEN || github.token }} - uses: shivammathur/setup-php@v2 with: @@ -58,37 +74,16 @@ jobs: # The Makefile includes files of the Composer packages. - run: composer install --no-interaction --prefer-dist - - name: Choose the version + - name: Choose the baseline id: version env: - INPUT_VERSION: ${{ inputs.version }} INPUT_BASELINE: ${{ inputs.baseline }} run: | latest=$(gh release view --json tagName --jq .tagName) - IFS=. read -r major minor patch <<< "${latest}" - if [ -n "${INPUT_VERSION}" ]; then - version="${INPUT_VERSION}" - else - # Y when a file of INSEE or La Poste is newer than the latest release, Z when only the BAN can have moved. - sources_out="${RUNNER_TEMP}/sources.out" - : > "${sources_out}" - GITHUB_OUTPUT="${sources_out}" python3 .github/scripts/check_sources.py - if grep -q '^changed=true' "${sources_out}"; then - version="${major}.$((minor + 1)).0" - echo "A source of INSEE or La Poste is newer: minor version." - else - version="${major}.${minor}.$((patch + 1))" - echo "No source of INSEE or La Poste is newer: patch version, for the points of the BAN." - fi - fi - if gh release list --limit 200 --json tagName --jq '.[].tagName' | grep -qx "${version}"; then - echo "::error::The release ${version} already exists, published or draft." - exit 1 - fi baseline="${INPUT_BASELINE:-${latest}}" + echo "latest=${latest}" >> "${GITHUB_OUTPUT}" echo "baseline=${baseline}" >> "${GITHUB_OUTPUT}" - echo "version=${version}" >> "${GITHUB_OUTPUT}" - echo "Latest release ${latest}, starting from ${baseline}, building ${version}." + echo "Latest release ${latest}, starting from ${baseline}." - name: Start the database run: | @@ -140,16 +135,12 @@ jobs: fi echo "No identifier changed: $(wc -l < identifiers-before.txt) before, $(wc -l < identifiers-after.txt) after." - - name: Export and zip the release files - env: - VERSION: ${{ steps.version.outputs.version }} + - name: Export the files run: | # The container writes the exports and the runner writes the SQL dump and the zips: share the directory. mkdir -p storage/app/exports/sql chmod -R a+rwX storage/app/exports make export - make release-files VERSION="${VERSION}" - ${COMPOSE} exec -T php php artisan dataset:status --no-ansi | tee status.txt # The schemas are read by data.gouv.fr: a file that does not respect them must not be published. - name: Check the files against their schemas @@ -175,24 +166,85 @@ jobs: cat compare.md } >> "${GITHUB_STEP_SUMMARY}" + - name: Choose the version and the action + id: next + env: + INPUT_VERSION: ${{ inputs.version }} + PUBLISH_PATCH: ${{ inputs.publish_patch }} + LATEST: ${{ steps.version.outputs.latest }} + CHANGED: ${{ steps.compare.outputs.changed }} + KIND: ${{ steps.compare.outputs.kind }} + REVIEW: ${{ steps.compare.outputs.review }} + CAN_PUBLISH: ${{ secrets.RELEASE_TOKEN != '' && secrets.GPG_PRIVATE_KEY_B64 != '' }} + run: | + IFS=. read -r major minor patch <<< "${LATEST}" + case "${KIND}" in + major) version="$((major + 1)).0.0" ;; + minor) version="${major}.$((minor + 1)).0" ;; + *) version="${major}.${minor}.$((patch + 1))" ;; + esac + + # none: nothing to release. hold: a patch kept for the day of the control. draft: to review. publish: by itself. + if [ "${CHANGED}" != "true" ]; then + action=none + elif [ -n "${INPUT_VERSION}" ]; then + version="${INPUT_VERSION}" + action=draft + elif [ "${REVIEW}" = "true" ]; then + action=draft + elif [ "${KIND}" = "patch" ] && [ "${PUBLISH_PATCH}" != "true" ]; then + action=hold + elif [ "${CAN_PUBLISH}" != "true" ]; then + echo "::warning::RELEASE_TOKEN or GPG_PRIVATE_KEY_B64 is not set: the release stays a draft." + action=draft + else + action=publish + fi + + if gh release list --limit 200 --json tagName --jq '.[].tagName' | grep -qx "${version}"; then + echo "::error::The release ${version} already exists, published or draft." + exit 1 + fi + echo "version=${version}" >> "${GITHUB_OUTPUT}" + echo "action=${action}" >> "${GITHUB_OUTPUT}" + echo "Latest release ${LATEST}, kind of change ${KIND}: ${version}, action ${action}." + + - name: Zip the release files + if: steps.next.outputs.action != 'none' && steps.next.outputs.action != 'hold' + env: + VERSION: ${{ steps.next.outputs.version }} + run: | + make release-files VERSION="${VERSION}" + ${COMPOSE} exec -T php php artisan dataset:status --no-ansi | tee status.txt + - uses: actions/upload-artifact@v7 + if: steps.next.outputs.action != 'none' && steps.next.outputs.action != 'hold' with: name: release-files path: storage/app/exports/release/* retention-days: 14 - - name: Create the draft release - if: ${{ !inputs.dry_run && steps.compare.outputs.changed == 'true' }} + - name: Write the notes of the release + if: ${{ !inputs.dry_run && (steps.next.outputs.action == 'draft' || steps.next.outputs.action == 'publish') }} env: - VERSION: ${{ steps.version.outputs.version }} + VERSION: ${{ steps.next.outputs.version }} BASELINE: ${{ steps.version.outputs.baseline }} + KIND: ${{ steps.compare.outputs.kind }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ACTION: ${{ steps.next.outputs.action }} + REASON: ${{ steps.compare.outputs.review_reason }} run: | { - echo "Built by the workflow Build dataset (${RUN_URL}) from the SQL of ${BASELINE}." - echo - echo "Check the files, then publish this draft: that creates the tag ${VERSION} on main and starts the publication on data.gouv.fr." + echo "Built by the workflow Build dataset (${RUN_URL}) from the SQL of ${BASELINE}: ${KIND} change." echo + if [ "${ACTION}" = "draft" ]; then + echo "Check the files, then publish this draft: that creates the tag ${VERSION} on main and starts the publication on data.gouv.fr." + if [ -n "${REASON}" ]; then + echo + echo "To review: ${REASON}." + fi + echo + fi echo "## Changes since ${BASELINE}" echo cat compare.md @@ -201,30 +253,87 @@ jobs: cat status.txt echo '```' } > notes.md + + - name: Create the draft release + if: ${{ !inputs.dry_run && steps.next.outputs.action == 'draft' }} + env: + VERSION: ${{ steps.next.outputs.version }} + REASON: ${{ steps.compare.outputs.review_reason }} + run: | gh release create "${VERSION}" --draft --target main --title "French-postal-code-${VERSION}" \ --notes-file notes.md --generate-notes storage/app/exports/release/* number=$(gh issue list --label data-update --state open --json number --jq '.[0].number // empty') if [ -n "${number}" ]; then - gh issue comment "${number}" --body "A draft release ${VERSION} is ready: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases. Check it and publish it." + gh issue comment "${number}" --body "A draft release ${VERSION} is ready and has to be reviewed: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases. Check it and publish it." fi + curl -fsS -m 15 -H "Title: French-Postal-Code ${VERSION} to review" -H "Priority: high" \ + -d "A draft release waits for a review: ${REASON:-the structure of the files changed}. ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases" \ + "${NTFY_URL}" > /dev/null || echo "The ntfy alert could not be sent." + + # The tag is signed with the key of the bot and pushed by the maintainer token: the rules of the tags only let a + # maintainer create them. The release is created by the same token, so that its publication starts the other workflows + # (a release created with the token of the workflow would start none). + - name: Publish the release + if: ${{ !inputs.dry_run && steps.next.outputs.action == 'publish' }} + env: + GH_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GPG_PRIVATE_KEY_B64: ${{ secrets.GPG_PRIVATE_KEY_B64 }} + SIGNER_NAME: ${{ secrets.RELEASE_SIGNER_NAME }} + SIGNER_EMAIL: ${{ secrets.RELEASE_SIGNER_EMAIL }} + VERSION: ${{ steps.next.outputs.version }} + KIND: ${{ steps.compare.outputs.kind }} + run: | + if [ -z "${SIGNER_NAME}" ] || [ -z "${SIGNER_EMAIL}" ]; then + echo "::error::RELEASE_SIGNER_NAME or RELEASE_SIGNER_EMAIL is not set." + exit 1 + fi + printf '%s' "${GPG_PRIVATE_KEY_B64}" | base64 -d | gpg --batch --import + key=$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr/ {print $10; exit}') + git config user.name "${SIGNER_NAME}" + git config user.email "${SIGNER_EMAIL}" + git config user.signingkey "${key}" + + git fetch --quiet origin main + git tag -s "${VERSION}" -m "${VERSION}" -m "Dataset ${KIND} release." origin/main + git tag -v "${VERSION}" + git push origin "${VERSION}" - - name: Report that the data did not change - if: ${{ !inputs.dry_run && steps.compare.outputs.changed == 'false' }} + gh release create "${VERSION}" --verify-tag --latest --title "French-postal-code-${VERSION}" \ + --notes-file notes.md --generate-notes storage/app/exports/release/* + + curl -fsS -m 15 -H "Title: French-Postal-Code ${VERSION} published" -H "Priority: low" \ + -d "${KIND} release published by itself. ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/tag/${VERSION}" \ + "${NTFY_URL}" > /dev/null || echo "The ntfy message could not be sent." + + - name: Report that there is no release + if: ${{ !inputs.dry_run && (steps.next.outputs.action == 'none' || steps.next.outputs.action == 'hold') }} env: FINGERPRINT: ${{ inputs.fingerprint }} BASELINE: ${{ steps.version.outputs.baseline }} + ACTION: ${{ steps.next.outputs.action }} run: | number=$(gh issue list --label data-update --state open --json number --jq '.[0].number // empty') if [ -z "${number}" ]; then exit 0 fi { - echo "The build ran on the new sources: the data are identical to ${BASELINE}, in every column, so there is no draft." - echo "The sources are not built again until they change. Close this issue if you do not expect a release." + if [ "${ACTION}" = "hold" ]; then + echo "The build ran on the new sources: only the points of the BAN changed since ${BASELINE}, a patch release, which is published on Monday." + else + echo "The build ran on the new sources: the data are identical to ${BASELINE}, in every column, so there is no release." + echo "The sources are not built again until they change. Close this issue if you do not expect a release." + fi if [ -n "${FINGERPRINT}" ]; then echo echo "" fi } > comment.md gh issue comment "${number}" --body-file comment.md + + - name: Tell that the build failed + if: ${{ failure() && !inputs.dry_run }} + run: | + curl -fsS -m 15 -H "Title: French-Postal-Code build failed" -H "Priority: high" \ + -d "The workflow Build dataset failed: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + "${NTFY_URL}" > /dev/null || echo "The ntfy alert could not be sent." diff --git a/.github/workflows/watch-sources.yml b/.github/workflows/watch-sources.yml index 711041e..dcd2269 100644 --- a/.github/workflows/watch-sources.yml +++ b/.github/workflows/watch-sources.yml @@ -1,11 +1,13 @@ name: Watch sources # Every day, compare the sources (INSEE COG, La Poste) with the time of the last generation of the dataset. -# When one is newer, open an issue that says so and start the workflow `Build dataset`, which proposes the new -# version as a draft release when the data changed: the maintainer only reviews and publishes it. +# When one is newer, open an issue that says so and start the workflow `Build dataset`, which publishes the new +# version by itself when the data changed (a minor release), or proposes a draft to review (see docs/release.md). # # The points of the BAN move without any of these files changing: every Monday (or with `force`), the build is -# started even when no source is newer. It creates a draft, in patch version, only when its export differs. +# started even when no source is newer, and may publish a patch release. The other days, a patch change waits. +# +# A draft waits for the maintainer: it blocks the builds, and a notification reminds of it every day. on: schedule: - cron: '30 5 * * *' @@ -20,7 +22,7 @@ on: default: false permissions: - contents: read + contents: write issues: write actions: write @@ -61,6 +63,21 @@ jobs: gh issue create --title "A source of the dataset changed: build a new version" --body-file body.md --label data-update --assignee Stanislas-Poisson fi + - name: Remind of the drafts to review + if: inputs.since == '' + env: + GH_TOKEN: ${{ github.token }} + NTFY_URL: ${{ secrets.NTFY_URL }} + run: | + gh release list --limit 50 --json tagName,isDraft,createdAt \ + --jq '.[] | select(.isDraft) | "\(.tagName) \(.createdAt)"' | while read -r tag created; do + days=$(( ($(date -u +%s) - $(date -u -d "${created}" +%s)) / 86400 )) + echo "The draft ${tag} waits for a review since ${days} day(s)." + curl -fsS -m 15 -H "Title: French-Postal-Code ${tag} waits for a review" -H "Priority: high" \ + -d "The draft has waited ${days} day(s) and blocks the new builds: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases" \ + "${NTFY_URL}" > /dev/null || echo "The ntfy reminder could not be sent." + done + - name: Start the build if: inputs.since == '' env: @@ -70,22 +87,27 @@ jobs: FORCE: ${{ inputs.force }} run: | fingerprint="" + control=false + if [ "$(date -u +%u)" = "1" ] || [ "${FORCE}" = "true" ]; then + control=true + fi if [ "${CHANGED}" = "true" ]; then - # The same sources are built once: a build that finds no change leaves a marker in the issue. + # The same sources are built once: a build that finds nothing to release leaves a marker in the issue. + # The weekly control builds them again, because a patch release is published on that day. fingerprint=$(printf '%s' "${REASONS}" | sha256sum | cut -c1-16) number=$(gh issue list --label data-update --state open --json number --jq '.[0].number // empty') - if [ -n "${number}" ] && gh issue view "${number}" --json comments --jq '.comments[].body' | grep -q "build-checked:${fingerprint}"; then - echo "These sources were already built: the data did not change." + if [ "${control}" != "true" ] && [ -n "${number}" ] && gh issue view "${number}" --json comments --jq '.comments[].body' | grep -q "build-checked:${fingerprint}"; then + echo "These sources were already built: nothing to release before the weekly control." exit 0 fi - elif [ "$(date -u +%u)" = "1" ] || [ "${FORCE}" = "true" ]; then + elif [ "${control}" = "true" ]; then echo "Weekly control of the BAN: build even though no source is newer." else echo "No source is newer, and it is not the day of the control of the BAN." exit 0 fi - # One build at a time, and none while a draft release waits for the maintainer. + # One build at a time, and none while a draft release waits for the review of the maintainer. drafts=$(gh release list --limit 50 --json isDraft --jq 'map(select(.isDraft)) | length') running=$(gh run list --workflow build-dataset.yml --json status --jq 'map(select(.status != "completed")) | length') if [ "${drafts}" != "0" ] || [ "${running}" != "0" ]; then @@ -94,6 +116,9 @@ jobs: fi args=(-f dry_run=false) + if [ "${control}" = "true" ]; then + args+=(-f publish_patch=true) + fi if [ -n "${fingerprint}" ]; then args+=(-f fingerprint="${fingerprint}") fi diff --git a/README.md b/README.md index e1d9ccc..59828e2 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ The matching `php artisan` commands: ## Published files -The exports are attached to the [releases][releases] of the repository and published on [data.gouv.fr][data-gouv]. They contain one file per table, in CSV, JSON and SQL. +The exports are attached to the [releases][releases] of the repository and published on [data.gouv.fr][data-gouv]. They contain one file per table, in CSV, JSON and SQL. A new version is published by itself, from the kind of change: **patch** when only the GPS points moved (at most once a week, on Monday), **minor** when anything else changed (at once), **major** when the structure of the files changed (reviewed by hand, see [`docs/release.md`][release]). ![Content of the dataset][img-dataset] diff --git a/docs/release.md b/docs/release.md index 854f9a2..499a91d 100644 --- a/docs/release.md +++ b/docs/release.md @@ -2,7 +2,7 @@ The dataset files are produced by `make export` and attached to the GitHub release. They are not committed to the repository. -A release is built by the workflow `Build dataset`, or by hand with the steps below. The workflow proposes a draft release: the maintainer reviews it and publishes it. +A release is built by the workflow `Build dataset`, or by hand with the steps below. The workflow publishes the release by itself when the change is a minor or a patch one, and proposes a draft release to review when it is a major one or when many rows were removed. ## Automatic build @@ -12,14 +12,22 @@ The workflow `Build dataset` (Actions tab, or started by `Watch sources` when a 2. It stops if the schema of the restored tables is not the one of the migrations. After a migration that changes these tables, build the baseline by hand once. 3. It runs the update, then stops if an identifier of the restored data changed or disappeared (regions, departments, communes, cities, commune successions and reference changes: the package loads the files by identifier), or if the update is incomplete. 4. It exports and checks the CSV files against the Table Schemas of the branch `schemas` (it stops if one does not conform), then writes the release files (the version is the next minor one when a source of INSEE or La Poste is newer, the next patch one otherwise, see [Versioning](#versioning); `version` overrides it) and keeps them as an artifact for 14 days. -5. It compares the export with the CSV files of the baseline release, in every column (the points of the BAN are part of the dataset): - - if anything differs, it creates a **draft** release with the files, a summary of the changes (rows added, removed, modified and the columns concerned), the notes and the status of the dataset, and comments the issue "A source of the dataset changed"; - - if nothing differs, it creates no draft: a newer source file does not mean that the data changed. It comments the issue and leaves a marker, so that `Watch sources` does not build the same sources again. +5. It compares the export with the CSV files of the baseline release, in every column (the points of the BAN are part of the dataset), and gives the **kind of change** (see [Versioning](#versioning)): + - nothing differs: it creates no release. A newer source file does not mean that the data changed. It comments the issue and leaves a marker, so that `Watch sources` does not build the same sources again; + - patch (only the points of the BAN moved): the release is **published by itself on Monday** (the control of the BAN, input `publish_patch`). The other days it comments the issue and keeps the change for Monday; + - minor: the release is **published by itself**, the day it is built; + - major, or more than 1 percent of the rows of a file removed: it creates a **draft** release with the files, a summary of the changes (rows added, removed, modified and the columns concerned), the notes and the status of the dataset, comments the issue, and sends a notification (ntfy) because the maintainer has to review it. -Started by hand, `dry_run` is on by default: the build and the checks run, but no draft is created. Check the draft, then publish it: that is the only manual step. The publication creates the tag on `main` and starts `Publish on data.gouv.fr`, which updates the resources, closes the issue of the sources and starts the update of French-Postal-Code-Package. The package then follows by itself: the pull request of the new data is merged when its CI passes, `develop` is brought to `main`, and the next minor version is tagged (signed) and published, see the README of the package. It stops, and says why, if the layout of the files changed: that is a major release, tagged by hand. +Started by hand, `dry_run` is on by default: the build and the checks run, but nothing is created or sent. A version given by hand (`version`) is always proposed as a draft. + +Publishing creates the tag `X.Y.Z` on `main`, signed with the key of the bot, then the release. The rules of the tags only let a maintainer create them, so the workflow uses the secret `RELEASE_TOKEN` (a token of the maintainer, which also starts the next workflows: a release created with the token of the workflow would start none). Without `RELEASE_TOKEN` or `GPG_PRIVATE_KEY_B64`, the release stays a draft. The publication starts `Publish on data.gouv.fr` (which updates the resources, closes the issue of the sources and starts the update of French-Postal-Code-Package) and `Stamp schemas`. The package then follows by itself: the pull request of the new data is merged when its CI passes, `develop` is brought to `main`, and the next version is tagged (signed) and published, see the README of the package. It stops, and says why, if the layout of the files changed: that is a major release, tagged by hand. + +The notifications go to ntfy (the secret `NTFY_URL`): a draft to review, a build that failed, and a daily reminder of a draft that waits, because a draft blocks the new builds. A release published by itself sends a message with a low priority. The secret `PACKAGE_DISPATCH_TOKEN` (a token with the right Actions: write on French-Postal-Code-Package) lets the publication start that update at once. Without it, the package finds the release on its weekly run. +The secrets of the repository are `RELEASE_TOKEN`, `GPG_PRIVATE_KEY_B64`, `RELEASE_SIGNER_NAME` and `RELEASE_SIGNER_EMAIL` (the bot that signs the tags, the same as for the base tools), `NTFY_URL`, `DATAGOUV_API_KEY` and `PACKAGE_DISPATCH_TOKEN`. + ## Steps by hand 1. Update the dataset and export the files in one go. @@ -67,17 +75,17 @@ The workflow `Watch sources` runs every day (and from the Actions tab). It compa - INSEE COG: a newer vintage, or a file of the latest vintage that changed after the generation (read on the dataset of data.gouv.fr, because INSEE sends no `Last-Modified`); - La Poste: the `Last-Modified` header of the file of postal codes. -When a source is newer it opens the issue "A source of the dataset changed: build a new version" (label `data-update`), updates it every day until a release is made, and starts the workflow `Build dataset` unless a draft release is waiting or a build is running. When nothing changed it does nothing, except every Monday (and with the input `force`): the points of the BAN move without any of these files changing, so the build is started anyway. It creates a draft, in patch version, only when its export differs from the latest release. The update works on the database that holds the history and the identifiers of the dataset, which is why the build restores the SQL of the latest release first: an empty database would give new identifiers. +When a source is newer it opens the issue "A source of the dataset changed: build a new version" (label `data-update`), updates it every day until a release is made, and starts the workflow `Build dataset` unless a draft release is waiting or a build is running. When nothing changed it does nothing, except every Monday (and with the input `force`): the points of the BAN move without any of these files changing, so the build is started anyway, and a patch release may be published. The same sources are built once during the week: the other days a build that finds nothing to publish leaves a marker in the issue. A draft release that waits is reminded every day by a notification. The update works on the database that holds the history and the identifiers of the dataset, which is why the build restores the SQL of the latest release first: an empty database would give new identifiers. ## Versioning -The version number follows SemVer, from what changed: +The version number follows SemVer, from what the export changed compared to the latest release (the script `.github/scripts/compare_exports.py` decides): -- **Z** (4.0.0 to 4.0.1): only the points of the BAN changed (`latitude`, `longitude`, `address_count`), the files of INSEE and La Poste did not. -- **Y** (4.0.1 to 4.1.0): a file of INSEE (COG) or La Poste is newer. Z goes back to 0. -- **X** (4.1.0 to 5.0.0): the schema of the files changed, for example a column removed. Tagged by hand. +- **Z** (4.0.0 to 4.0.1): only the points of the BAN changed, that is `address_count`, `latitude` and `longitude` of `cities.csv`. Published on Monday. +- **Y** (4.0.1 to 4.1.0): anything else in the rows: a row added or removed, or another column modified, in any file. Z goes back to 0. Published at once. +- **X** (4.1.0 to 5.0.0): the structure of the files changed (the columns of a file, a file added or removed). Always a draft, then tagged by hand after the change of the schemas. -The workflow `Build dataset` chooses between Y and Z: it asks the check of `Watch sources` whether a source is newer than the latest release. The `version` input overrides it. French-Postal-Code-Package follows the same rule from the release of the builder its data come from. +French-Postal-Code-Package follows the same rule from the release of the builder its data come from. data.gouv.fr is updated at each release: at most once a week for the points of the BAN, and when a source changes for the rest. ## Statistics cards