From d5987150716d585a17980ad491c04dbdeb229269 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 19:14:58 +0200 Subject: [PATCH 01/16] update pylint to v3.3.* Remove the now redundant controls, found with: pylint --enable=useless-suppression --- .github/workflows/tests.yml | 2 +- .pylintrc | 3 --- conda_env/gdal-dev.yml | 2 +- wahoomc/osm_maps_functions.py | 7 +++---- wahoomc/setup_functions.py | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2f778b49..9a9ed87a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pylint==2.15.* + pip install pylint==3.3.* pip install mock pip install packaging pip install requests==2.28.* diff --git a/.pylintrc b/.pylintrc index a95f775e..22dd6143 100644 --- a/.pylintrc +++ b/.pylintrc @@ -17,6 +17,3 @@ disable=line-too-long, duplicate-code ; [MASTER] ; init-hook='import sys; sys.path.append("/path/to/root")' ; init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" - -[MASTER] -init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" diff --git a/conda_env/gdal-dev.yml b/conda_env/gdal-dev.yml index 89f124d0..8e651f2d 100644 --- a/conda_env/gdal-dev.yml +++ b/conda_env/gdal-dev.yml @@ -5,7 +5,7 @@ dependencies: - python=3.10 - gdal=3.6.* - requests=2.28.* - - pylint=2.15.* + - pylint=3.3.* - geojson=2.5.* - shapely=1.8.* - osmium-tool=1.16.* diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index a46c17f5..d47b8623 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -49,7 +49,7 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): ) else: - process = subprocess.run( # pylint: disable=consider-using-with + process = subprocess.run( cmd, capture_output=True, cwd=cwd, @@ -59,7 +59,6 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): check=False, ) - if error_message and process.returncode != 0: # 0 means success log.error('subprocess error output:') if process.stderr: @@ -463,7 +462,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto log.info('# Merge splitted tiles with land, elevation, and sea') timings = Timings() tile_count = 1 - for tile in self.o_osm_data.tiles: # pylint: disable=too-many-nested-blocks + for tile in self.o_osm_data.tiles: self.log_tile_info(tile["x"], tile["y"], tile_count) timings_tile = Timings() @@ -816,7 +815,7 @@ def log_tile_debug(self, tile_x, tile_y, tile_count, additional_info=''): """ self.log_tile(tile_x, tile_y, tile_count, True, additional_info) - def log_tile(self, tile_x, tile_y, tile_count, log_level_debug, additional_info=''): # pylint: disable=too-many-arguments + def log_tile(self, tile_x, tile_y, tile_count, log_level_debug, additional_info=''): # pylint: disable=too-many-arguments,too-many-positional-arguments """ unified status logging for this class """ diff --git a/wahoomc/setup_functions.py b/wahoomc/setup_functions.py index 2b181f58..d9fc7766 100644 --- a/wahoomc/setup_functions.py +++ b/wahoomc/setup_functions.py @@ -45,7 +45,7 @@ def adjustments_due_to_breaking_changes(): """ handle breaking changes """ - version_last_run = read_version_last_run() # pylint: disable=unused-variable + version_last_run = read_version_last_run() # Osmosis in v.0.49.2 seams not to be working on WINDOWS since the upgrade to v0.49.2 # - due to the path into it was downloaded, 'tooling_win/Osmosis/osmosis-0.49.2' From ab72557b990d8f4e1c091db8e945cb801efdc8cb Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 19:15:48 +0200 Subject: [PATCH 02/16] fix pylint possibly-used-before-assignment warning is_required_input_given_or_exit() already checks that either country or x/y is set, make pylint happy with a simple "else". Fixes: wahoomc/main.py:76:8: E0606: Possibly using variable 'o_osm_data' before assignment (possibly-used-before-assignment) --- tests/test_osm_maps.py | 3 ++- wahoomc/main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index f061ad0d..6a9569a7 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -204,7 +204,8 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, if inp_mode == 'country': o_input_data.country = inp_val o_osm_data = CountryOsmData(o_input_data) - elif inp_mode == 'xy_coordinate': + else: + self.assertEqual(inp_mode, 'xy_coordinate') o_input_data.xy_coordinates = inp_val o_osm_data = XYOsmData(o_input_data) diff --git a/wahoomc/main.py b/wahoomc/main.py index 0f516111..d29d6de8 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -61,7 +61,7 @@ def run(run_level): if o_input_data.country: o_osm_data = CountryOsmData(o_input_data) - elif o_input_data.xy_coordinates: + else: o_osm_data = XYOsmData(o_input_data) timings = Timings() From 92cc017515747ed4003fed0982d26d7be18faa0f Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Mon, 19 May 2025 10:49:49 +0200 Subject: [PATCH 03/16] use the current python interpreter for additional processes This makes it work everywhere, no matter if it's `python` or `python3`. --- tests/test_cli.py | 7 ++++--- tests/test_generated_files.py | 5 +++-- wahoomc/osm_maps_functions.py | 12 ++---------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index c2723198..aa595524 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,7 @@ tests for the python file """ import os +import sys import unittest # import custom python packages @@ -17,7 +18,7 @@ def test_top_parser_help(self): tests, if help of top parser can be called """ - result = os.system("python -m wahoomc -h") + result = os.system(sys.executable + " -m wahoomc -h") self.assertEqual(result, 0) @@ -26,7 +27,7 @@ def test_cli_help(self): tests, if CLI help can be called """ - result = os.system("python -m wahoomc cli -h") + result = os.system(sys.executable + " -m wahoomc cli -h") self.assertEqual(result, 0) @@ -35,7 +36,7 @@ def test_gui_help(self): tests, if GUI help can be called """ - result = os.system("python -m wahoomc gui -h") + result = os.system(sys.executable + " -m wahoomc gui -h") self.assertEqual(result, 0) diff --git a/tests/test_generated_files.py b/tests/test_generated_files.py index bd314205..aac7dffb 100644 --- a/tests/test_generated_files.py +++ b/tests/test_generated_files.py @@ -7,6 +7,7 @@ from os import walk import platform import shutil +import sys import unittest import subprocess @@ -214,11 +215,11 @@ def run_wahoomapscreator_cli(self, country, hdd_mode=False): if not hdd_mode: # run processing of input-country via CLI in standard mode result = os.system( - f'python -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc') else: # run processing of input-country via CLI in mapwriter hdd mode result = os.system( - f'python -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hdd') + f'{sys.executable} -m wahoomc cli -co {country} -tag tag-wahoo.xml -fp -c -md 9999 -nbc -hdd') # check if run was successful self.assertEqual(result, 0) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index d47b8623..247955da 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -253,16 +253,8 @@ def generate_land(self): # create land1.osm if not os.path.isfile(out_file_land1+'1.osm') or self.o_osm_data.force_processing is True: - # Windows - if platform.system() == "Windows": - cmd = ['python', os.path.join(RESOURCES_DIR, - 'shape2osm.py'), '-l', out_file_land1, land_file] - - # Non-Windows - else: - cmd = ['python', os.path.join(RESOURCES_DIR, - 'shape2osm.py'), '-l', out_file_land1, land_file] - + cmd = [sys.executable, os.path.join(RESOURCES_DIR, + 'shape2osm.py'), '-l', out_file_land1, land_file] run_subprocess_and_log_output( cmd, f'! Error creating land.osm for tile: {tile["x"]},{tile["y"]}') self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return()) From 445487ebe4b1e7dcd564f2fc9ac5dc8e7b4dd017 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Sun, 25 May 2025 08:29:24 +0200 Subject: [PATCH 04/16] catch JSONDecodeError in get_latest_pypi_version() This actually happened, in which case everything falls apart. --- wahoomc/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index bed2d948..338f4fc4 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -180,7 +180,7 @@ def get_latest_pypi_version(): response = requests.get( 'https://pypi.org/pypi/wahoomc/json', timeout=1) return response.json()['info']['version'] - except (requests.ConnectionError, requests.Timeout): + except (requests.ConnectionError, requests.Timeout, requests.exceptions.JSONDecodeError): return None From 8da9d5bcc1cee5be06347fe6f6ab54b3c574546b Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Mon, 2 Jun 2025 18:58:13 +0200 Subject: [PATCH 05/16] log full subprocess command lines Visible when using --verbose. --- wahoomc/osm_maps_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 247955da..731e9741 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -38,6 +38,7 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): """ run given cmd-subprocess and issue error message if wished """ + log.debug('running subprocess: %s', str(cmd)) if not cwd: process = subprocess.run( cmd, From 4defb0ab9e295aa656f28f2388bbb3dbe48b2dc6 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Mon, 2 Jun 2025 18:57:24 +0200 Subject: [PATCH 06/16] make translate_tags_to_keep() platform independent Pass on which tool to use, osmfilter or osmium. This makes it easier to switch to the other tool, as all are available on all platforms. --- tests/test_constants.py | 10 ++++------ wahoomc/constants_functions.py | 20 ++++++-------------- wahoomc/osm_maps_functions.py | 28 ++++++++++------------------ 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 8c6c5e82..1c9b74d7 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -51,7 +51,7 @@ def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # tags_win = 'access= area=yes' mock_json_load.return_value = tags_universal_simple - transl_tags = translate_tags_to_keep(sys_platform='Windows') + transl_tags = translate_tags_to_keep(osmium=False) self.assertEqual(tags_win, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -76,7 +76,7 @@ def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pyl tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' mock_json_load.return_value = tags_universal_adv - transl_tags = translate_tags_to_keep(sys_platform='Windows') + transl_tags = translate_tags_to_keep(osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_tags_to_keep_full_macos(self): @@ -100,8 +100,7 @@ def test_translate_tags_to_keep_full_win(self): """ tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated amenity=fuel =cafe =drinking_water =shelter shop=bakery =bicycle highway=abandoned =bus_guideway =disused =bridleway =byway =construction =cycleway =footway =living_street =motorway =motorway_link =path =pedestrian =primary =primary_link =residential =road =secondary =secondary_link =service =steps =tertiary =tertiary_link =track =trunk =trunk_link =unclassified natural=coastline =nosea =sea =beach =land =scrub =water =wetland =wood landuse=forest =commercial =industrial =residential =retail leisure=park =nature_reserve railway=rail =tram =station =stop surface= tracktype= tunnel= waterway=canal =drain =river =riverbank =stream wood=deciduous tourism=alpine_hut' - transl_tags = translate_tags_to_keep( - sys_platform='Windows', use_repo=True) + transl_tags = translate_tags_to_keep(osmium=False, use_repo=True) self.assertEqual(tags_win, transl_tags) def test_translate_name_tags_to_keep_full_macos(self): @@ -121,8 +120,7 @@ def test_translate_name_tags_to_keep_full_win(self): names_tags_win = 'admin_level=2 area=yes mountain_pass= natural= place=city =hamlet =island =isolated_dwelling =islet =locality =suburb =town =village =country' - transl_tags = translate_tags_to_keep( - name_tags=True, sys_platform='Windows', use_repo=True) + transl_tags = translate_tags_to_keep(name_tags=True, osmium=False, use_repo=True) self.assertEqual(names_tags_win, transl_tags) diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 012b8e3e..6852d592 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -26,16 +26,10 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" -def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): +def translate_tags_to_keep(name_tags=False, osmium=True, use_repo=False): """ translates the given tags to format of the operating system. """ - - if sys_platform == "Windows": - separator = ' =' - else: - separator = ', ' - tags_modif = [] # read tags-to-keep .json from user-dir in favor of python installation @@ -61,21 +55,22 @@ def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): universal_tags = tags_from_json['NAME_TAGS_TO_KEEP_UNIVERSAL'] for tag, value in universal_tags.items(): - to_append = transl_tag_value(sys_platform, separator, tag, value) + to_append = transl_tag_value(osmium, tag, value) tags_modif.append(to_append) - if sys_platform == "Windows": + if not osmium: tags_modif = ' '.join(tags_modif) return tags_modif -def transl_tag_value(sys_platform, separator, tag, value): +def transl_tag_value(osmium, tag, value): """ translates one tag with value(s) to a "common" format """ if isinstance(value, list): + separator = ', ' if osmium else ' =' for iteration, sing_val in enumerate(value): if iteration == 0: to_append = f'{tag}={sing_val}' @@ -84,10 +79,7 @@ def transl_tag_value(sys_platform, separator, tag, value): elif value: to_append = f'{tag}={value}' else: - if sys_platform == "Windows": - to_append = f'{tag}=' - else: - to_append = tag + to_append = tag if osmium else f'{tag}=' return to_append diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 731e9741..fb1a480a 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -146,10 +146,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= layer= ' + - translate_tags_to_keep(sys_platform=platform.system()) + '"') + cmd.append('--keep="' + translate_tags_to_keep(osmium=False) + '"') + cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_win) run_subprocess_and_log_output( @@ -157,12 +155,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= name= layer= ' + - translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') + cmd.append('--keep="' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') + cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_names_win) run_subprocess_and_log_output( @@ -189,8 +183,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep( - sys_platform=platform.system())) + cmd.extend(translate_tags_to_keep()) cmd.extend(['-o', out_file_pbf_filtered_mac]) cmd.append('--overwrite') @@ -199,8 +192,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep( - name_tags=True, sys_platform=platform.system())) + cmd.extend(translate_tags_to_keep(name_tags=True)) cmd.extend(['-o', out_file_pbf_filtered_names_mac]) cmd.append('--overwrite') @@ -756,8 +748,8 @@ def write_country_config_file(self, country): configuration = { "version_last_run": VERSION, "changed_ts_map_last_run": get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']), - "tags_last_run": translate_tags_to_keep(sys_platform=platform.system()), - "name_tags_last_run": translate_tags_to_keep(name_tags=True, sys_platform=platform.system()) + "tags_last_run": translate_tags_to_keep(), + "name_tags_last_run": translate_tags_to_keep(name_tags=True) } write_json_file_generic(os.path.join( @@ -772,8 +764,8 @@ def tags_are_identical_to_last_run(self, country): try: country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) - if not country_config["tags_last_run"] == translate_tags_to_keep(sys_platform=platform.system()) \ - or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True, sys_platform=platform.system()): + if not country_config["tags_last_run"] == translate_tags_to_keep() \ + or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True): tags_are_identical = False except (FileNotFoundError, KeyError): tags_are_identical = False From 03d9e2aca523d1d55f9c5d4fb4d1300c4ddd867f Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:16:53 +0200 Subject: [PATCH 07/16] get rid of unused issue_message argument It wasn't used consistently anyway. --- wahoomc/input.py | 11 ++++------- wahoomc/main.py | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/wahoomc/input.py b/wahoomc/input.py index 3f95afcc..7e168173 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -196,7 +196,7 @@ def __init__(self): self.verbose = False - def is_required_input_given_or_exit(self, issue_message): + def is_required_input_given_or_exit(self): """ check, if the minimal required arguments is given: - country @@ -205,11 +205,8 @@ def is_required_input_given_or_exit(self, issue_message): If not, depending on the import parameter, the """ if (self.country in ('None', '') and self.xy_coordinates in ('None', '')): - if issue_message: - sys.exit("Nothing to do. Start with -h or --help to see command line options." - "Or in the GUI select a country to create maps for.") - else: - sys.exit() + sys.exit("Nothing to do. Start with -h or --help to see command line options." + "Or in the GUI select a country to create maps for.") elif self.country and self.xy_coordinates: sys.exit( "Country and X/Y coordinates are given. Only one of both is allowed!") @@ -245,7 +242,7 @@ def start_gui(self): # start GUI self.mainloop() - self.o_input_data.is_required_input_given_or_exit(issue_message=True) + self.o_input_data.is_required_input_given_or_exit() return self.o_input_data def build_gui(self): diff --git a/wahoomc/main.py b/wahoomc/main.py index d29d6de8..d9a67cf0 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -54,7 +54,7 @@ def run(run_level): copy_jsons_from_repo_to_user('.', 'tags-to-keep.json') else: # Is there something to do? - o_input_data.is_required_input_given_or_exit(issue_message=True) + o_input_data.is_required_input_given_or_exit() if o_input_data.contour: check_installation_of_programs_credentials_for_contour_lines() From 747754dc6731f997d8817314d3a432e689164cac Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:19:05 +0200 Subject: [PATCH 08/16] clean up is_required_input_given_or_exit() No need for "else" if an error case sys.exit()s. --- wahoomc/input.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/wahoomc/input.py b/wahoomc/input.py index 7e168173..ee8955c6 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -196,7 +196,8 @@ def __init__(self): self.verbose = False - def is_required_input_given_or_exit(self): + + def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches """ check, if the minimal required arguments is given: - country @@ -207,20 +208,18 @@ def is_required_input_given_or_exit(self): if (self.country in ('None', '') and self.xy_coordinates in ('None', '')): sys.exit("Nothing to do. Start with -h or --help to see command line options." "Or in the GUI select a country to create maps for.") - elif self.country and self.xy_coordinates: - sys.exit( - "Country and X/Y coordinates are given. Only one of both is allowed!") - elif self.country: + + if self.country and self.xy_coordinates: + sys.exit("Country and X/Y coordinates are given. Only one of both is allowed!") + + if self.country: # countries = try: CountryGeofabrik.split_input_to_list(self.country) except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) - # if we made it until here, sys.exit() was not called and therefore all countries OK ;-) - return True - else: - return True + return True class GuiInput(tk.Tk): From cdd94a29b5488dd73338e3fd622567bb00ef2f7b Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:26:54 +0200 Subject: [PATCH 09/16] allow setting an arbitrary --tag_wahoo_xml file Don't just look in the user config directory and the bundled resources/ folder, allow passing any filename to --tag_wahoo_xml. Order of preference: - filesystem, either relative or absolute - relative to the user config directory - relative to the bundled resources directory --- tests/test_constants.py | 28 +--------------------------- wahoomc/constants_functions.py | 19 ------------------- wahoomc/input.py | 13 ++++++++++++- wahoomc/osm_maps_functions.py | 14 +++----------- 4 files changed, 16 insertions(+), 58 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 1c9b74d7..05660015 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,14 +1,11 @@ """ tests for the downloader file """ -import os import unittest import mock -from wahoomc.constants_functions import translate_tags_to_keep, \ - get_tag_wahoo_xml_path, TagWahooXmlNotFoundError -from wahoomc.constants import RESOURCES_DIR +from wahoomc.constants_functions import translate_tags_to_keep tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": { @@ -124,28 +121,5 @@ def test_translate_name_tags_to_keep_full_win(self): self.assertEqual(names_tags_win, transl_tags) -class TestTagWahooXML(unittest.TestCase): - """ - tests for tag-wahoo xml file - """ - - def test_not_existing_tag_wahoo_xml(self): - """ - check if a non-existing tag-wahoo xml file issues an exception - """ - with self.assertRaises(TagWahooXmlNotFoundError): - get_tag_wahoo_xml_path("not_existing.xml") - - def test_existing_tag_wahoo_xml(self): - """ - check if the correct path of an existing tag-wahoo xml file is returned - """ - - expected_path = os.path.join( - RESOURCES_DIR, "tag_wahoo_adjusted", "tag-wahoo-poi.xml") - self.assertEqual(get_tag_wahoo_xml_path( - "tag-wahoo-poi.xml"), expected_path) - - if __name__ == '__main__': unittest.main() diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 6852d592..3160fd15 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -18,10 +18,6 @@ log = logging.getLogger('main-logger') -class TagWahooXmlNotFoundError(Exception): - """Raised when the specified tag-wahoo xml file does not exist""" - - class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" @@ -104,21 +100,6 @@ def get_tooling_win_path(path_in_tooling_win, in_user_dir=False): # all other "toolings": concatenate with win tooling dir return os.path.join(tooling_dir, path_in_tooling_win) - -def get_tag_wahoo_xml_path(tag_wahoo_xml): - """ - return path to tag-wahoo xml file if the file exists - - from the user directory "USER_WAHOO_MC/_config/tag_wahoo_adjusted/tag_wahoo_xml" - - 2ndly from the PyPI installation: "RESOURCES_DIR/tag_wahoo_adjusted/tag_wahoo_xml" - """ - - for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", tag_wahoo_xml): - if os.path.exists(path): - return path - - raise TagWahooXmlNotFoundError - - def get_absolute_dir_user_or_repo(folder, file=''): """ return the absolute path to the folder (and file) in this priorization diff --git a/wahoomc/input.py b/wahoomc/input.py index ee8955c6..c762a29b 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -5,6 +5,7 @@ # import official python packages import argparse +import os import sys # for gui @@ -12,6 +13,7 @@ from tkinter import ttk # import custom python packages +from wahoomc.constants_functions import get_absolute_dir_user_or_repo from wahoomc.geofabrik_json import GeofabrikJson from wahoomc.geofabrik_json import CountyIsNoGeofabrikCountry from wahoomc.geofabrik import CountryGeofabrik @@ -77,7 +79,7 @@ def process_call_of_the_tool(): # Save uncompressed maps for Cruiser if True options_args.add_argument('-c', '--cruiser', action='store_true', help="save uncompressed maps for Cruiser") - # specify the file with tags to keep in the output // file needs to be in wahoo_mc/resources/tag_wahoo_adjusted + # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") # zip the country (and country-maps) folder @@ -219,6 +221,15 @@ def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) + if not os.path.exists(self.tag_wahoo_xml): + for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): + if os.path.exists(path): + self.tag_wahoo_xml = path + break + + if not os.path.exists(self.tag_wahoo_xml): + sys.exit(f'The tag-wahoo xml file was not found: \"{self.tag_wahoo_xml}\"') + return True diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index fb1a480a..daaacb60 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -16,8 +16,7 @@ # import custom python packages from wahoomc.file_directory_functions import read_json_file_country_config, create_empty_directories, write_json_file_generic -from wahoomc.constants_functions import translate_tags_to_keep, \ - get_tooling_win_path, get_tag_wahoo_xml_path, TagWahooXmlNotFoundError +from wahoomc.constants_functions import translate_tags_to_keep, get_tooling_win_path from wahoomc.setup_functions import read_earthexplorer_credentials @@ -554,7 +553,7 @@ def sort_osm_files(self, tile, verbose): log.debug('+ Sorting land* osm files: OK') - def create_map_files(self, save_cruiser, tag_wahoo_xml, hdd_mode, verbose): + def create_map_files(self, save_cruiser, tag_conf_file, hdd_mode, verbose): """ Creating .map files """ @@ -595,14 +594,7 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml, hdd_mode, verbose): cmd.append(f'threads={threads}') if hdd_mode: cmd.append('type=hd') - # add path to tag-wahoo xml file - try: - cmd.append( - f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}') - except TagWahooXmlNotFoundError: - log.error( - 'The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml) - sys.exit() + cmd.append(f'tag-conf-file={tag_conf_file}') if verbose: result = subprocess.run(cmd, check=False) From 0f378176106f2626b4936aff4cb773cae30e72b1 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 11:57:17 +0200 Subject: [PATCH 10/16] log the used map-writer tag-conf file --- wahoomc/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wahoomc/main.py b/wahoomc/main.py index d9a67cf0..e9caf3bd 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -18,6 +18,8 @@ from wahoomc.osm_maps_functions import OsmMaps from wahoomc.osm_data import CountryOsmData, XYOsmData +log = logging.getLogger('main-logger') + # logging used in the terminal output: # # means top-level command # ! means error @@ -47,7 +49,7 @@ def run(run_level): o_input_data = process_call_of_the_tool() if o_input_data.verbose: - logging.getLogger().setLevel(logging.DEBUG) + log.setLevel(logging.DEBUG) if run_level == 'init': copy_jsons_from_repo_to_user('tag_wahoo_adjusted') @@ -59,6 +61,9 @@ def run(run_level): if o_input_data.contour: check_installation_of_programs_credentials_for_contour_lines() + log.info('# Used configuration') + log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) + if o_input_data.country: o_osm_data = CountryOsmData(o_input_data) else: From fb6568a63a9ca703b95639836b59a51814fb9a91 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 12:36:12 +0200 Subject: [PATCH 11/16] add --tags_to_keep to set an arbitrary tags-to-keep.json file Use the same logic as with --tag_wahoo_xml --- tests/test_constants.py | 21 +++++++++++++-------- tests/test_osm_maps.py | 19 ++++++++++++++----- wahoomc/constants_functions.py | 17 +++-------------- wahoomc/input.py | 14 ++++++++++++++ wahoomc/main.py | 3 ++- wahoomc/osm_maps_functions.py | 23 ++++++++++++----------- 6 files changed, 58 insertions(+), 39 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 05660015..cc158355 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,10 +1,12 @@ """ tests for the downloader file """ +import os import unittest import mock +from wahoomc.constants import RESOURCES_DIR from wahoomc.constants_functions import translate_tags_to_keep @@ -27,6 +29,9 @@ class TestTranslateTags(unittest.TestCase): tests for translating tags-constants between the universal format and OS-specific formats """ + def setUp(self): + self.tags_to_keep = os.path.join(RESOURCES_DIR, 'tags-to-keep.json') + @ mock.patch("wahoomc.file_directory_functions.json.load") @ mock.patch("wahoomc.open") def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument @@ -36,7 +41,7 @@ def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): tags = ['access', 'area=yes'] mock_json_load.return_value = tags_universal_simple - transl_tags = translate_tags_to_keep() + transl_tags = translate_tags_to_keep('nonexistant.json') self.assertEqual(tags, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -48,7 +53,7 @@ def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # tags_win = 'access= area=yes' mock_json_load.return_value = tags_universal_simple - transl_tags = translate_tags_to_keep(osmium=False) + transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) self.assertEqual(tags_win, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -61,7 +66,7 @@ def test_translate_tags_to_keep_adv_macos(self, mock_open, mock_json_load): # p 'bridge', 'foot=ft_yes, foot_designated'] mock_json_load.return_value = tags_universal_adv - transl_tags = translate_tags_to_keep() + transl_tags = translate_tags_to_keep('nonexistant.json') self.assertEqual(tags, transl_tags) @ mock.patch("wahoomc.file_directory_functions.json.load") @@ -73,7 +78,7 @@ def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pyl tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' mock_json_load.return_value = tags_universal_adv - transl_tags = translate_tags_to_keep(osmium=False) + transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_tags_to_keep_full_macos(self): @@ -88,7 +93,7 @@ def test_translate_tags_to_keep_full_macos(self): 'leisure=park, nature_reserve', 'railway=rail, tram, station, stop', 'surface', 'tracktype', 'tunnel', 'waterway=canal, drain, river, riverbank, stream', 'wood=deciduous', 'tourism=alpine_hut'] - transl_tags = translate_tags_to_keep(use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep) self.assertEqual(tags, transl_tags) def test_translate_tags_to_keep_full_win(self): @@ -97,7 +102,7 @@ def test_translate_tags_to_keep_full_win(self): """ tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated amenity=fuel =cafe =drinking_water =shelter shop=bakery =bicycle highway=abandoned =bus_guideway =disused =bridleway =byway =construction =cycleway =footway =living_street =motorway =motorway_link =path =pedestrian =primary =primary_link =residential =road =secondary =secondary_link =service =steps =tertiary =tertiary_link =track =trunk =trunk_link =unclassified natural=coastline =nosea =sea =beach =land =scrub =water =wetland =wood landuse=forest =commercial =industrial =residential =retail leisure=park =nature_reserve railway=rail =tram =station =stop surface= tracktype= tunnel= waterway=canal =drain =river =riverbank =stream wood=deciduous tourism=alpine_hut' - transl_tags = translate_tags_to_keep(osmium=False, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, osmium=False) self.assertEqual(tags_win, transl_tags) def test_translate_name_tags_to_keep_full_macos(self): @@ -107,7 +112,7 @@ def test_translate_name_tags_to_keep_full_macos(self): names_tags = ['admin_level=2', 'area=yes', 'mountain_pass', 'natural', 'place=city, hamlet, island, isolated_dwelling, islet, locality, suburb, town, village, country'] - transl_tags = translate_tags_to_keep(name_tags=True, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, name_tags=True) self.assertEqual(names_tags, transl_tags) def test_translate_name_tags_to_keep_full_win(self): @@ -117,7 +122,7 @@ def test_translate_name_tags_to_keep_full_win(self): names_tags_win = 'admin_level=2 area=yes mountain_pass= natural= place=city =hamlet =island =isolated_dwelling =islet =locality =suburb =town =village =country' - transl_tags = translate_tags_to_keep(name_tags=True, osmium=False, use_repo=True) + transl_tags = translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) self.assertEqual(names_tags_win, transl_tags) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 6a9569a7..60c45a99 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -62,7 +62,16 @@ def test_subprocess_output_decode_error_handler_with_cwd(self, mock_run): ) -class TestOsmMapsCalculation(unittest.TestCase): +class TestOsmMaps(unittest.TestCase): + """ + base test class + """ + + def setUp(self): + self.tags_to_keep = os.path.join(constants.RESOURCES_DIR, 'tags-to-keep.json') + + +class TestOsmMapsCalculation(TestOsmMaps): """ tests for the OSM maps file """ @@ -220,7 +229,7 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, self.assertEqual(result, exp_result) -class TestOSMMapsInput(unittest.TestCase): +class TestOSMMapsInput(TestOsmMaps): """ tests for input of OsmData """ @@ -241,7 +250,7 @@ def test_folder_name_many_countries(self): """ o_osm_data = self.get_osm_data_instance('albania,alps,andorra,austria,azores,belarus,belgium,bosnia-herzegovina,britain-and-ireland,bulgaria,croatia,cyprus,czech-republic,dach,denmark,estonia,faroe-islands,finland,france,georgia,germany,great-britain,greece,guernsey-jersey,hungary,iceland,ireland-and-northern-ireland,isle-of-man,italy,kosovo,latvia,liechtenstein,lithuania,luxembourg,macedonia,malta,moldova,monaco,montenegro,netherlands,norway,poland,portugal,romania,serbia,slovakia,slovenia,spain,sweden,switzerland,turkey,ukraine') - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) folder_name = o_osm_maps.calculate_folder_name('.map.lzma') folder_name_maps = o_osm_maps.calculate_folder_name('.map') @@ -272,7 +281,7 @@ def get_osm_data_instance(self, country_input): return o_osm_data -class TestConfigFile(unittest.TestCase): +class TestConfigFile(TestOsmMaps): """ tests for the config .json file in the "wahooMapsCreatorData/_tiles/{country}" directory """ @@ -295,7 +304,7 @@ def test_version_and_tags_of_country_config_file(self): # download files marked for download to fill up map_file per country to write to config o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) o_osm_maps.write_country_config_file(o_input_data.country) diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 3160fd15..53e2d59d 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -22,25 +22,14 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" -def translate_tags_to_keep(name_tags=False, osmium=True, use_repo=False): +def translate_tags_to_keep(tags_to_keep, name_tags=False, osmium=True): """ translates the given tags to format of the operating system. """ tags_modif = [] - # read tags-to-keep .json from user-dir in favor of python installation - # evaluate path first: user-dir in favor of PyPI installation - if not use_repo: - for path in get_absolute_dir_user_or_repo('', file='tags-to-keep.json'): - if os.path.exists(path): - break - # force using file from repo - used in unittests for equal output - else: - path = get_absolute_dir_user_or_repo( - '', file='tags-to-keep.json')[1] - - # read the tags from the evaluated path above - tags_from_json = read_json_file_generic(path) + # read the tags from the passed tags_to_keep path + tags_from_json = read_json_file_generic(tags_to_keep) if not tags_from_json: raise TagsToKeepNotFoundError diff --git a/wahoomc/input.py b/wahoomc/input.py index c762a29b..2c887e9b 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -79,6 +79,9 @@ def process_call_of_the_tool(): # Save uncompressed maps for Cruiser if True options_args.add_argument('-c', '--cruiser', action='store_true', help="save uncompressed maps for Cruiser") + # specify the file with tags to keep when filtering + options_args.add_argument('--tags_to_keep', default=InputData().tags_to_keep, + help="file with tags to keep when filtering") # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") @@ -111,6 +114,7 @@ def process_call_of_the_tool(): o_input_data.force_download = args.forcedownload o_input_data.force_processing = args.forceprocessing + o_input_data.tags_to_keep = args.tags_to_keep o_input_data.tag_wahoo_xml = args.tag_wahoo_xml o_input_data.save_cruiser = args.cruiser o_input_data.zip_folder = args.zip @@ -191,6 +195,7 @@ def __init__(self): self.contour = False self.use_srtm1 = False + self.tags_to_keep = "tags-to-keep.json" self.tag_wahoo_xml = "tag-wahoo-poi.xml" self.zip_folder = False self.save_cruiser = False @@ -221,6 +226,15 @@ def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches except CountyIsNoGeofabrikCountry as exception: sys.exit(exception) + if not os.path.exists(self.tags_to_keep): + for path in get_absolute_dir_user_or_repo("", self.tags_to_keep): + if os.path.exists(path): + self.tags_to_keep = path + break + + if not os.path.exists(self.tags_to_keep): + sys.exit(f'The tags-to-keep json file was not found: \"{self.tags_to_keep}\"') + if not os.path.exists(self.tag_wahoo_xml): for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): if os.path.exists(path): diff --git a/wahoomc/main.py b/wahoomc/main.py index e9caf3bd..0a12b527 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -62,6 +62,7 @@ def run(run_level): check_installation_of_programs_credentials_for_contour_lines() log.info('# Used configuration') + log.info('+ tags-to-keep file: %s', o_input_data.tags_to_keep) log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) if o_input_data.country: @@ -77,7 +78,7 @@ def run(run_level): # Download files marked for download o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data) + o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep) # Filter tags from country osm.pbf files' o_osm_maps.filter_tags_from_country_osm_pbf_files() diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index daaacb60..7bbfe10e 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -89,8 +89,9 @@ class OsmMaps: # Number of workers for the Osmosis read binary fast function workers = '1' - def __init__(self, o_osm_data): + def __init__(self, o_osm_data, tags_to_keep): self.o_osm_data = o_osm_data + self.tags_to_keep = tags_to_keep self.osmconvert_path = get_tooling_win_path('osmconvert') create_empty_directories( @@ -145,8 +146,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append('--keep="' + translate_tags_to_keep(osmium=False) + '"') - cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(osmium=False) + '"') + cmd.append('--keep="' + translate_tags_to_keep(self.tags_to_keep, osmium=False) + '"') + cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(self.tags_to_keep, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_win) run_subprocess_and_log_output( @@ -154,8 +155,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)] cmd.append(out_file_o5m) - cmd.append('--keep="' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') - cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(name_tags=True, osmium=False) + '"') + cmd.append('--keep="' + translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) + '"') + cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep(self.tags_to_keep, name_tags=True, osmium=False) + '"') cmd.append('-o=' + out_file_o5m_filtered_names_win) run_subprocess_and_log_output( @@ -182,7 +183,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep()) + cmd.extend(translate_tags_to_keep(self.tags_to_keep)) cmd.extend(['-o', out_file_pbf_filtered_mac]) cmd.append('--overwrite') @@ -191,7 +192,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st cmd = ['osmium', 'tags-filter', '--remove-tags'] cmd.append(val['map_file']) - cmd.extend(translate_tags_to_keep(name_tags=True)) + cmd.extend(translate_tags_to_keep(self.tags_to_keep, name_tags=True)) cmd.extend(['-o', out_file_pbf_filtered_names_mac]) cmd.append('--overwrite') @@ -740,8 +741,8 @@ def write_country_config_file(self, country): configuration = { "version_last_run": VERSION, "changed_ts_map_last_run": get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']), - "tags_last_run": translate_tags_to_keep(), - "name_tags_last_run": translate_tags_to_keep(name_tags=True) + "tags_last_run": translate_tags_to_keep(self.tags_to_keep), + "name_tags_last_run": translate_tags_to_keep(self.tags_to_keep, name_tags=True) } write_json_file_generic(os.path.join( @@ -756,8 +757,8 @@ def tags_are_identical_to_last_run(self, country): try: country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) - if not country_config["tags_last_run"] == translate_tags_to_keep() \ - or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True): + if not country_config["tags_last_run"] == translate_tags_to_keep(self.tags_to_keep) \ + or not country_config["name_tags_last_run"] == translate_tags_to_keep(self.tags_to_keep, name_tags=True): tags_are_identical = False except (FileNotFoundError, KeyError): tags_are_identical = False From b63957dabc64565b392497367126df761ca49388 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 22:44:10 +0200 Subject: [PATCH 12/16] copy the current tags-to-keep file to tests/ And test that instead. There's no need to adapt the test everytime the file gets changed. --- tests/resources/tags-to-keep.json | 109 ++++++++++++++++++++++++++++++ tests/test_constants.py | 3 +- 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 tests/resources/tags-to-keep.json diff --git a/tests/resources/tags-to-keep.json b/tests/resources/tags-to-keep.json new file mode 100644 index 00000000..b306b27c --- /dev/null +++ b/tests/resources/tags-to-keep.json @@ -0,0 +1,109 @@ +{ + "TAGS_TO_KEEP_UNIVERSAL": { + "access": "", + "area": "yes", + "bicycle": "", + "bridge": "", + "foot": [ + "ft_yes", + "foot_designated" + ], + "amenity": [ + "fuel", + "cafe", + "drinking_water", + "shelter" + ], + "shop": [ + "bakery", + "bicycle" + ], + "highway": [ + "abandoned", + "bus_guideway", + "disused", + "bridleway", + "byway", + "construction", + "cycleway", + "footway", + "living_street", + "motorway", + "motorway_link", + "path", + "pedestrian", + "primary", + "primary_link", + "residential", + "road", + "secondary", + "secondary_link", + "service", + "steps", + "tertiary", + "tertiary_link", + "track", + "trunk", + "trunk_link", + "unclassified" + ], + "natural": [ + "coastline", + "nosea", + "sea", + "beach", + "land", + "scrub", + "water", + "wetland", + "wood" + ], + "landuse": [ + "forest", + "commercial", + "industrial", + "residential", + "retail" + ], + "leisure": [ + "park", + "nature_reserve" + ], + "railway": [ + "rail", + "tram", + "station", + "stop" + ], + "surface": "", + "tracktype": "", + "tunnel": "", + "waterway": [ + "canal", + "drain", + "river", + "riverbank", + "stream" + ], + "wood": "deciduous", + "tourism": "alpine_hut" + }, + "NAME_TAGS_TO_KEEP_UNIVERSAL": { + "admin_level": "2", + "area": "yes", + "mountain_pass": "", + "natural": "", + "place": [ + "city", + "hamlet", + "island", + "isolated_dwelling", + "islet", + "locality", + "suburb", + "town", + "village", + "country" + ] + } +} \ No newline at end of file diff --git a/tests/test_constants.py b/tests/test_constants.py index cc158355..d5e23eeb 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -6,7 +6,6 @@ import mock -from wahoomc.constants import RESOURCES_DIR from wahoomc.constants_functions import translate_tags_to_keep @@ -30,7 +29,7 @@ class TestTranslateTags(unittest.TestCase): """ def setUp(self): - self.tags_to_keep = os.path.join(RESOURCES_DIR, 'tags-to-keep.json') + self.tags_to_keep = os.path.join(os.path.dirname(__file__), 'resources', 'tags-to-keep.json') @ mock.patch("wahoomc.file_directory_functions.json.load") @ mock.patch("wahoomc.open") From b41b29f8b2691c8cc0f3b7170f866332cf18f40c Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 22:45:31 +0200 Subject: [PATCH 13/16] get rid of mock The functionality is already tested with real files, there's no need to bend over backwards and hack around with mock, which doesn't work in its current state anyway. --- .github/workflows/tests.yml | 1 - conda_env/gdal-dev.yml | 1 - tests/test_constants.py | 64 ------------------------------------- 3 files changed, 66 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a9ed87a..9e50fc4b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,6 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint==3.3.* - pip install mock pip install packaging pip install requests==2.28.* - name: Analysing the code with pylint diff --git a/conda_env/gdal-dev.yml b/conda_env/gdal-dev.yml index 8e651f2d..027c7e96 100644 --- a/conda_env/gdal-dev.yml +++ b/conda_env/gdal-dev.yml @@ -14,7 +14,6 @@ dependencies: - matplotlib=3.4.3 - packaging - autopep8=2.0.* - - mock - twine - pip - vulture diff --git a/tests/test_constants.py b/tests/test_constants.py index d5e23eeb..0039364c 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -3,26 +3,11 @@ """ import os import unittest -import mock from wahoomc.constants_functions import translate_tags_to_keep -tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": { - 'access': '', - 'area': 'yes' -}} - -tags_universal_adv = {"TAGS_TO_KEEP_UNIVERSAL": { - 'access': '', - 'area': 'yes', - 'bicycle': '', - 'bridge': '', - 'foot': ['ft_yes', 'foot_designated'] -}} - - class TestTranslateTags(unittest.TestCase): """ tests for translating tags-constants between the universal format and OS-specific formats @@ -31,55 +16,6 @@ class TestTranslateTags(unittest.TestCase): def setUp(self): self.tags_to_keep = os.path.join(os.path.dirname(__file__), 'resources', 'tags-to-keep.json') - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_simple_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to macOS - """ - tags = ['access', 'area=yes'] - mock_json_load.return_value = tags_universal_simple - - transl_tags = translate_tags_to_keep('nonexistant.json') - self.assertEqual(tags, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_simple_win(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to Windows - """ - tags_win = 'access= area=yes' - mock_json_load.return_value = tags_universal_simple - - transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) - self.assertEqual(tags_win, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_adv_macos(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to macOS - """ - tags = ['access', 'area=yes', 'bicycle', - 'bridge', 'foot=ft_yes, foot_designated'] - mock_json_load.return_value = tags_universal_adv - - transl_tags = translate_tags_to_keep('nonexistant.json') - self.assertEqual(tags, transl_tags) - - @ mock.patch("wahoomc.file_directory_functions.json.load") - @ mock.patch("wahoomc.open") - def test_translate_tags_to_keep_adv_win(self, mock_open, mock_json_load): # pylint: disable=unused-argument - """ - Test translating tags to keep from universal format to Windows - """ - tags_win = 'access= area=yes bicycle= bridge= foot=ft_yes =foot_designated' - mock_json_load.return_value = tags_universal_adv - - transl_tags = translate_tags_to_keep('nonexistant.json', osmium=False) - self.assertEqual(tags_win, transl_tags) - def test_translate_tags_to_keep_full_macos(self): """ Test translating tags to keep from universal format to macOS // all "tags to keep" From d8506e846524aa01f5df2e249320731a3a71f66d Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 5 Jun 2025 12:57:27 +0200 Subject: [PATCH 14/16] add --tag_transform to set an arbitrary tag-transform file Use the same logic as with --tag_wahoo_xml --- tests/test_osm_maps.py | 5 +++-- wahoomc/input.py | 14 ++++++++++++++ wahoomc/main.py | 3 ++- wahoomc/osm_maps_functions.py | 6 +++--- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 60c45a99..5aead7b8 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -69,6 +69,7 @@ class TestOsmMaps(unittest.TestCase): def setUp(self): self.tags_to_keep = os.path.join(constants.RESOURCES_DIR, 'tags-to-keep.json') + self.tag_transform = os.path.join(constants.RESOURCES_DIR, 'tunnel-transform.xml') class TestOsmMapsCalculation(TestOsmMaps): @@ -250,7 +251,7 @@ def test_folder_name_many_countries(self): """ o_osm_data = self.get_osm_data_instance('albania,alps,andorra,austria,azores,belarus,belgium,bosnia-herzegovina,britain-and-ireland,bulgaria,croatia,cyprus,czech-republic,dach,denmark,estonia,faroe-islands,finland,france,georgia,germany,great-britain,greece,guernsey-jersey,hungary,iceland,ireland-and-northern-ireland,isle-of-man,italy,kosovo,latvia,liechtenstein,lithuania,luxembourg,macedonia,malta,moldova,monaco,montenegro,netherlands,norway,poland,portugal,romania,serbia,slovakia,slovenia,spain,sweden,switzerland,turkey,ukraine') - o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep, self.tag_transform) folder_name = o_osm_maps.calculate_folder_name('.map.lzma') folder_name_maps = o_osm_maps.calculate_folder_name('.map') @@ -304,7 +305,7 @@ def test_version_and_tags_of_country_config_file(self): # download files marked for download to fill up map_file per country to write to config o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep) + o_osm_maps = OsmMaps(o_osm_data, self.tags_to_keep, self.tag_transform) o_osm_maps.write_country_config_file(o_input_data.country) diff --git a/wahoomc/input.py b/wahoomc/input.py index 2c887e9b..bf71e5c8 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -82,6 +82,9 @@ def process_call_of_the_tool(): # specify the file with tags to keep when filtering options_args.add_argument('--tags_to_keep', default=InputData().tags_to_keep, help="file with tags to keep when filtering") + # specify the file with tag-transform rules + options_args.add_argument('--tag_transform', default=InputData().tag_transform, + help="file with tag-transform rules") # specify the file with tags to keep in the output options_args.add_argument('-tag', '--tag_wahoo_xml', default=InputData().tag_wahoo_xml, help="file with tags to keep in the output") @@ -115,6 +118,7 @@ def process_call_of_the_tool(): o_input_data.force_processing = args.forceprocessing o_input_data.tags_to_keep = args.tags_to_keep + o_input_data.tag_transform = args.tag_transform o_input_data.tag_wahoo_xml = args.tag_wahoo_xml o_input_data.save_cruiser = args.cruiser o_input_data.zip_folder = args.zip @@ -196,6 +200,7 @@ def __init__(self): self.use_srtm1 = False self.tags_to_keep = "tags-to-keep.json" + self.tag_transform = "tunnel-transform.xml" self.tag_wahoo_xml = "tag-wahoo-poi.xml" self.zip_folder = False self.save_cruiser = False @@ -235,6 +240,15 @@ def is_required_input_given_or_exit(self): # pylint: disable=too-many-branches if not os.path.exists(self.tags_to_keep): sys.exit(f'The tags-to-keep json file was not found: \"{self.tags_to_keep}\"') + if not os.path.exists(self.tag_transform): + for path in get_absolute_dir_user_or_repo("", self.tag_transform): + if os.path.exists(path): + self.tag_transform = path + break + + if not os.path.exists(self.tag_transform): + sys.exit(f'The tag-transform xml file was not found: \"{self.tag_transform}\"') + if not os.path.exists(self.tag_wahoo_xml): for path in get_absolute_dir_user_or_repo("tag_wahoo_adjusted", self.tag_wahoo_xml): if os.path.exists(path): diff --git a/wahoomc/main.py b/wahoomc/main.py index 0a12b527..54bbf95b 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -63,6 +63,7 @@ def run(run_level): log.info('# Used configuration') log.info('+ tags-to-keep file: %s', o_input_data.tags_to_keep) + log.info('+ tag-transform file: %s', o_input_data.tag_transform) log.info('+ map-writer tag-conf file: %s', o_input_data.tag_wahoo_xml) if o_input_data.country: @@ -78,7 +79,7 @@ def run(run_level): # Download files marked for download o_downloader.download_files_if_needed() - o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep) + o_osm_maps = OsmMaps(o_osm_data, o_input_data.tags_to_keep, o_input_data.tag_transform) # Filter tags from country osm.pbf files' o_osm_maps.filter_tags_from_country_osm_pbf_files() diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 7bbfe10e..a6ab13f5 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -89,9 +89,10 @@ class OsmMaps: # Number of workers for the Osmosis read binary fast function workers = '1' - def __init__(self, o_osm_data, tags_to_keep): + def __init__(self, o_osm_data, tags_to_keep, tag_transform): self.o_osm_data = o_osm_data self.tags_to_keep = tags_to_keep + self.tag_transform = tag_transform self.osmconvert_path = get_tooling_win_path('osmconvert') create_empty_directories( @@ -502,8 +503,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto cmd.extend( ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m']) - cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR, - 'tunnel-transform.xml'), '--wb', out_file_merged, 'omitmetadata=true']) + cmd.extend(['--tag-transform', 'file=' + self.tag_transform, '--wb', out_file_merged, 'omitmetadata=true']) if verbose: result = subprocess.run(cmd, check=False) From 6f45d16ec01935a35e1262b12d62227dd82cc56b Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 22 May 2025 08:43:46 +0200 Subject: [PATCH 15/16] update dependency mapwriter plugin to 0.25.0 --- tests/test_downloader.py | 2 +- wahoomc/downloader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader.py b/tests/test_downloader.py index c8b3ba55..475ef49e 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -186,7 +186,7 @@ def test_download_macos_files(self): """ if platform.system() != "Windows": path = os.path.join(str(constants.USER_DIR), '.openstreetmap', 'osmosis', - 'plugins', 'mapsforge-map-writer-0.21.0-jar-with-dependencies.jar') + 'plugins', 'mapsforge-map-writer-0.25.0-jar-with-dependencies.jar') if os.path.exists(path): os.remove(path) diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index 338f4fc4..57372b9c 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -119,8 +119,8 @@ def download_tooling(): check here for new mapwriter plugin version: https://github.com/mapsforge/mapsforge """ - map_writer_filename = 'mapsforge-map-writer-0.21.0-jar-with-dependencies.jar' - mapwriter_plugin_url = 'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.21.0/' + map_writer_filename + map_writer_filename = 'mapsforge-map-writer-0.25.0-jar-with-dependencies.jar' + mapwriter_plugin_url = 'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.25.0/' + map_writer_filename # Windows if platform.system() == "Windows": From dbabe269a513926acb741aa8902af122f9746fa6 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Thu, 29 May 2025 08:29:16 +0200 Subject: [PATCH 16/16] enable tag-values for the mapsforge-map-writer plugin This enables wildcard values like %f and %s. The current ones in tag-wahoo-poi.xml were ignored unit now, so remove them to keep the results unchanged. --- wahoomc/osm_maps_functions.py | 1 + wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index a6ab13f5..ab2f06d7 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -593,6 +593,7 @@ def create_map_files(self, save_cruiser, tag_conf_file, hdd_mode, verbose): f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}') cmd.append('zoom-interval-conf=12,0,17') cmd.append(f'threads={threads}') + cmd.append('tag-values=true') if hdd_mode: cmd.append('type=hd') cmd.append(f'tag-conf-file={tag_conf_file}') diff --git a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml index 6f43434f..8fe06d45 100644 --- a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml +++ b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml @@ -171,11 +171,6 @@ - - - - -