Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions .github/scripts/compare_exports.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

- `changed` (true or false): every column counts, the points of the BAN are part of the dataset;
- `kind`: `none`, `patch`, `minor` or `major`, from what changed (see docs/release.md):
- `patch`: only the points of the BAN moved (`address_count`, `latitude`, `longitude` of cities.csv);
- `patch`: only the points of the BAN moved (`address_count`, `latitude`, `longitude` of cities.csv, and
`coordinate_source` going from `nominatim` to `ban` when a city gets its first point of the BAN);
- `minor`: anything else in the rows (a row added or removed, another column modified);
- `major`: the structure changed (the columns of a file, a file added or removed);
- `review` (true or false) and `review_reason`: a major change, or more than REMOVAL_THRESHOLD percent (1 by default) of
Expand All @@ -25,6 +26,8 @@

# The columns that move with the points of the BAN, without any file of INSEE or La Poste changing.
BAN_COLUMNS = {'cities.csv': {'address_count', 'latitude', 'longitude'}}
# A city that had no point of the BAN and gets one: its source of coordinates goes from Nominatim to the BAN.
BAN_SWITCHES = {('cities.csv', 'coordinate_source', 'nominatim', 'ban')}
DEFAULT_REMOVAL_THRESHOLD = 1.0


Expand All @@ -35,8 +38,16 @@ def read(path):
return header, list(reader)


def compare(old_path, new_path):
"""Returns (added, removed, modified, columns, header_changed, old_rows) for one file."""
def moves_with_ban(name, column, old_value, new_value):
"""True when a changed cell is a move of the points of the BAN (a patch change)."""
return column in BAN_COLUMNS.get(name, set()) or (name, column, old_value, new_value) in BAN_SWITCHES


def compare(name, old_path, new_path):
"""Returns (added, removed, modified, columns, blocking, header_changed, old_rows) for one file.

`columns` counts every modified column; `blocking` counts only those that are not a move of the BAN.
"""
old_header, old_rows = read(old_path)
new_header, new_rows = read(new_path)
header_changed = old_header != new_header
Expand All @@ -48,27 +59,29 @@ def compare(old_path, new_path):
added = len(new.keys() - old.keys())
removed = len(old.keys() - new.keys())
columns = Counter()
blocking = Counter()
modified = 0
for key in old.keys() & new.keys():
different = [name for name, a, b in zip(old_header, old[key], new[key]) if a != b]
if different:
changes = [(column, a, b) for column, a, b in zip(old_header, old[key], new[key]) if a != b]
if changes:
modified += 1
columns.update(different)
return added, removed, modified, columns, header_changed, len(old_rows)
columns.update(column for column, _, _ in changes)
blocking.update(column for column, a, b in changes if not moves_with_ban(name, column, a, b))
return added, removed, modified, columns, blocking, header_changed, len(old_rows)

old_set = Counter(map(tuple, old_rows))
new_set = Counter(map(tuple, new_rows))
return sum((new_set - old_set).values()), sum((old_set - new_set).values()), 0, Counter(), header_changed, len(old_rows)
return sum((new_set - old_set).values()), sum((old_set - new_set).values()), 0, Counter(), Counter(), header_changed, len(old_rows)


def classify(name, added, removed, modified, columns, header_changed):
def classify(added, removed, modified, blocking, header_changed):
"""The kind of change of one file: none, patch, minor or major."""
if header_changed:
return 'major'
if added or removed:
return 'minor'
if modified:
return 'patch' if set(columns) <= BAN_COLUMNS.get(name, set()) else 'minor'
return 'minor' if blocking else 'patch'
return 'none'


Expand All @@ -90,10 +103,10 @@ def main():
kind = 'major'
reasons.append(f"{name}: file {'added' if not old.exists() else 'removed'}")
continue
added, removed, modified, columns, header_changed, old_count = compare(old, new)
added, removed, modified, columns, blocking, header_changed, old_count = compare(name, old, new)
if added or removed or modified or header_changed:
changed = True
file_kind = classify(name, added, removed, modified, columns, header_changed)
file_kind = classify(added, removed, modified, blocking, header_changed)
kind = max(kind, file_kind, key=order.index)
if header_changed:
reasons.append(f'{name}: the columns changed')
Expand Down
2 changes: 1 addition & 1 deletion docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ When a source is newer it opens the issue "A source of the dataset changed: buil

The version number follows SemVer, from what the export changed compared to the latest release (the script `.github/scripts/compare_exports.py` decides):

- **Z** (4.0.0 to 4.0.1): only the points of the BAN changed, that is `address_count`, `latitude` and `longitude` of `cities.csv`. Published on Monday.
- **Z** (4.0.0 to 4.0.1): only the points of the BAN changed, that is `address_count`, `latitude` and `longitude` of `cities.csv`, and `coordinate_source` going from `nominatim` to `ban` when a city gets its first point of the BAN. Published on Monday.
- **Y** (4.0.1 to 4.1.0): anything else in the rows: a row added or removed, or another column modified, in any file. Z goes back to 0. Published at once.
- **X** (4.1.0 to 5.0.0): the structure of the files changed (the columns of a file, a file added or removed). Always a draft, then tagged by hand after the change of the schemas.

Expand Down
Loading