diff --git a/map_machine/geometry/bounding_box.py b/map_machine/geometry/bounding_box.py index 7b6e16a..14444f5 100644 --- a/map_machine/geometry/bounding_box.py +++ b/map_machine/geometry/bounding_box.py @@ -25,6 +25,16 @@ ) +def floor(value: float) -> float: + """Round down to 3 digits after the point.""" + return np.floor(value * 1000.0) / 1000.0 + + +def ceil(value: float) -> float: + """Round up to 3 digits after the point.""" + return np.ceil(value * 1000.0) / 1000.0 + + @dataclass class BoundingBox: """Rectangle that limits the space on the map.""" @@ -156,15 +166,25 @@ def get_format(self) -> str: """Get text representation of the bounding box. Bounding box format is - ,,,. Coordinates are + ,,,. Coordinates are rounded to three decimal places. """ - left: float = np.floor(self.left * 1000.0) / 1000.0 - bottom: float = np.floor(self.bottom * 1000.0) / 1000.0 - right: float = np.ceil(self.right * 1000.0) / 1000.0 - top: float = np.ceil(self.top * 1000.0) / 1000.0 + return ( + f"{floor(self.left):.3f},{floor(self.bottom):.3f}," + f"{ceil(self.right):.3f},{ceil(self.top):.3f}" + ) + + def get_overpass_format(self) -> str: + """Get bounding box in Overpass API format. - return f"{left:.3f},{bottom:.3f},{right:.3f},{top:.3f}" + Overpass bounding box format is ,,,, which + maps to ,,,. Minimum coordinates are floored + and maximum coordinates are ceiled to three decimal places. + """ + return ( + f"{floor(self.bottom):.3f},{floor(self.left):.3f}," + f"{ceil(self.top):.3f},{ceil(self.right):.3f}" + ) def update(self, coordinates: np.ndarray) -> None: """Make the bounding box cover coordinates.""" diff --git a/map_machine/mapper.py b/map_machine/mapper.py index eb2b798..dc6fbd4 100644 --- a/map_machine/mapper.py +++ b/map_machine/mapper.py @@ -32,6 +32,7 @@ NetworkError, find_incomplete_relations, get_osm, + get_osm_overpass, get_overpass_relations, ) from map_machine.osm.osm_reader import OSMData, OSMNode @@ -412,19 +413,41 @@ def render_map(arguments: argparse.Namespace) -> None: # Determine files. + overpass_query: str | None = None + overpass_query_path: str | None = getattr(arguments, "overpass_query", None) + if overpass_query_path: + with Path(overpass_query_path).open(encoding="utf-8") as query_file: + overpass_query = query_file.read() + input_file_names: list[Path] if arguments.input_file_names: input_file_names = list(map(Path, arguments.input_file_names)) elif bounding_box: - try: + overpass_cache_path: Path = ( + cache_path / f"{bounding_box.get_format()}_overpass.osm" + ) + if overpass_cache_path.is_file(): + input_file_names = [overpass_cache_path] + else: cache_file_path: Path = ( cache_path / f"{bounding_box.get_format()}.osm" ) - get_osm(bounding_box, cache_file_path) - input_file_names = [cache_file_path] - except NetworkError as error: - logger.fatal(error.message) - sys.exit(1) + try: + get_osm(bounding_box, cache_file_path) + input_file_names = [cache_file_path] + except NetworkError as error: + logger.warning( + "OSM API failed (%s), falling back to Overpass API...", + error.message, + ) + try: + get_osm_overpass( + bounding_box, overpass_cache_path, overpass_query + ) + input_file_names = [overpass_cache_path] + except NetworkError as overpass_error: + logger.fatal(overpass_error.message) + sys.exit(1) else: fatal( "Specify `--input`, `--bounding-box`, `--coordinates`, or `--gpx`." diff --git a/map_machine/osm/osm_getter.py b/map_machine/osm/osm_getter.py index 2000c98..2ead82b 100644 --- a/map_machine/osm/osm_getter.py +++ b/map_machine/osm/osm_getter.py @@ -7,6 +7,7 @@ import logging import time from dataclasses import dataclass +from textwrap import dedent from typing import TYPE_CHECKING import urllib3 @@ -26,6 +27,23 @@ MAX_OSM_MESSAGE_LENGTH: int = 500 OVERPASS_API_URL: str = "https://overpass-api.de/api/interpreter" +DEFAULT_OVERPASS_FILTER: str = dedent( + """ + ( + way["building"]; + way["highway"]; + way["natural"="water"]; + way["waterway"]; + way["natural"="wood"]; + way["landuse"="forest"]; + relation["natural"="water"]; + relation["landuse"="forest"]; + ); + (._;>;); + out body; + """ +) + @dataclass class NetworkError(Exception): @@ -158,3 +176,57 @@ def get_overpass_relations( cache_file.write_bytes(content) return content + + +def get_osm_overpass( + bounding_box: BoundingBox, + cache_file_path: Path, + query: str | None = None, + *, + to_update: bool = False, +) -> str: + """Download OSM data from the Overpass API. + + Uses a simplified filter query to fetch only major map features, which + avoids failures when the standard OSM API returns too much data. + + :param bounding_box: borders of the map part to download + :param cache_file_path: cache file to store downloaded OSM data + :param query: custom Overpass query with ``{{bbox}}`` placeholder; if None, + the default filter is used + :param to_update: update cache files + """ + if not to_update and cache_file_path.is_file(): + with cache_file_path.open(encoding="utf-8") as output_file: + return output_file.read() + + box: str = bounding_box.get_overpass_format() + + if query is not None: + full_query = query.replace("{{bbox}}", box) + else: + full_query = f"[out:xml][bbox:{box}];\n{DEFAULT_OVERPASS_FILTER}" + + logger.info("Querying Overpass API for bounding box %s...", box) + + try: + content: bytes = get_data(OVERPASS_API_URL, {"data": full_query}) + except NetworkError: + logger.warning("Failed to download data from Overpass API.") + raise + + if not content or not content.strip().startswith(b"<"): + if len(content) < MAX_OSM_MESSAGE_LENGTH: + message = ( + "Unexpected Overpass API response: `" + + content.decode("utf-8") + + "`." + ) + else: + message = "Unexpected Overpass API response." + raise NetworkError(message) + + with cache_file_path.open("bw+") as output_file: + output_file.write(content) + + return content.decode("utf-8") diff --git a/map_machine/slippy/tile.py b/map_machine/slippy/tile.py index 66d322b..81bcf1b 100644 --- a/map_machine/slippy/tile.py +++ b/map_machine/slippy/tile.py @@ -25,6 +25,7 @@ NetworkError, find_incomplete_relations, get_osm, + get_osm_overpass, get_overpass_relations, ) from map_machine.osm.osm_reader import OSMData @@ -141,15 +142,33 @@ def get_extended_bounding_box(self) -> BoundingBox: float(point_1[0]), ).round() - def load_osm_data(self, cache_path: Path) -> OSMData: + def load_osm_data( + self, cache_path: Path, overpass_query: str | None = None + ) -> OSMData: """Construct map data from extended bounding box. :param cache_path: directory to store OSM data files + :param overpass_query: custom Overpass query with `{{bbox}}` + placeholder """ - cache_file_path: Path = ( - cache_path / f"{self.get_extended_bounding_box().get_format()}.osm" + bounding_box: BoundingBox = self.get_extended_bounding_box() + overpass_cache_path: Path = ( + cache_path / f"{bounding_box.get_format()}_overpass.osm" ) - get_osm(self.get_extended_bounding_box(), cache_file_path) + + if overpass_cache_path.is_file(): + cache_file_path = overpass_cache_path + else: + cache_file_path = cache_path / f"{bounding_box.get_format()}.osm" + try: + get_osm(bounding_box, cache_file_path) + except NetworkError as error: + logger.warning( + "OSM API failed (%s), falling back to Overpass API...", + error.message, + ) + cache_file_path = overpass_cache_path + get_osm_overpass(bounding_box, cache_file_path, overpass_query) osm_data: OSMData = OSMData() osm_data.parse_osm_file(cache_file_path) @@ -178,6 +197,7 @@ def draw( configuration: MapConfiguration, *, use_overpass: bool = True, + overpass_query: str | None = None, ) -> None: """Draw tile to SVG and PNG files. @@ -185,9 +205,11 @@ def draw( :param cache_path: directory to store SVG and PNG tiles :param configuration: drawing configuration :param use_overpass: fetch missing relation data via Overpass API + :param overpass_query: custom Overpass query with `{{bbox}}` + placeholder """ try: - osm_data: OSMData = self.load_osm_data(cache_path) + osm_data: OSMData = self.load_osm_data(cache_path, overpass_query) except NetworkError as error: msg = f"Map is not loaded. {error.message}" raise NetworkError(msg) from error @@ -300,12 +322,35 @@ def from_bounding_box( return cls(tiles, tile_1, tile_2, zoom_level, extended_bounding_box) - def load_osm_data(self, cache_path: Path) -> OSMData: - """Load OpenStreetMap data.""" - cache_file_path: Path = ( - cache_path / f"{self.bounding_box.get_format()}.osm" + def load_osm_data( + self, cache_path: Path, overpass_query: str | None = None + ) -> OSMData: + """Load OpenStreetMap data. + + :param cache_path: directory for caching OSM data files + :param overpass_query: custom Overpass query with `{{bbox}}` placeholder + """ + overpass_cache_path: Path = ( + cache_path / f"{self.bounding_box.get_format()}_overpass.osm" ) - get_osm(self.bounding_box, cache_file_path) + + if overpass_cache_path.is_file(): + cache_file_path = overpass_cache_path + else: + cache_file_path = ( + cache_path / f"{self.bounding_box.get_format()}.osm" + ) + try: + get_osm(self.bounding_box, cache_file_path) + except NetworkError as error: + logger.warning( + "OSM API failed (%s), falling back to Overpass API...", + error.message, + ) + cache_file_path = overpass_cache_path + get_osm_overpass( + self.bounding_box, cache_file_path, overpass_query + ) osm_data: OSMData = OSMData() osm_data.parse_osm_file(cache_file_path) @@ -517,6 +562,12 @@ def generate_tiles(options: argparse.Namespace) -> None: use_overpass: bool = not getattr(options, "no_overpass", False) + overpass_query: str | None = None + overpass_query_path: str | None = getattr(options, "overpass_query", None) + if overpass_query_path: + with Path(overpass_query_path).open(encoding="utf-8") as query_file: + overpass_query = query_file.read() + if options.input_file_name: osm_data = OSMData() osm_data.parse_osm_file(Path(options.input_file_name)) @@ -548,7 +599,7 @@ def generate_tiles(options: argparse.Namespace) -> None: np.array(coordinates), min_zoom_level ) try: - osm_data = min_tile.load_osm_data(cache_path) + osm_data = min_tile.load_osm_data(cache_path, overpass_query) except NetworkError as error: message = f"Map is not loaded. {error.message}" raise NetworkError(message) from error @@ -577,6 +628,7 @@ def generate_tiles(options: argparse.Namespace) -> None: cache_path, configuration, use_overpass=use_overpass, + overpass_query=overpass_query, ) elif options.bounding_box: @@ -589,7 +641,7 @@ def generate_tiles(options: argparse.Namespace) -> None: min_tiles: Tiles = Tiles.from_bounding_box(bounding_box, min_zoom_level) try: - osm_data = min_tiles.load_osm_data(cache_path) + osm_data = min_tiles.load_osm_data(cache_path, overpass_query) except NetworkError as error: message = f"Map is not loaded. {error.message}" raise NetworkError(message) from error diff --git a/map_machine/ui/cli.py b/map_machine/ui/cli.py index fa33bcb..7a19084 100644 --- a/map_machine/ui/cli.py +++ b/map_machine/ui/cli.py @@ -278,6 +278,14 @@ def add_tile_arguments(parser: argparse.ArgumentParser) -> None: action="store_true", default=False, ) + parser.add_argument( + "--overpass-query", + metavar="", + help=( + "path to a custom Overpass query file; use {{bbox}} as a " + "placeholder for the bounding box" + ), + ) parser.add_argument( "-c", "--coordinates", @@ -370,6 +378,14 @@ def add_render_arguments(parser: argparse.ArgumentParser) -> None: action="store_true", default=False, ) + parser.add_argument( + "--overpass-query", + metavar="", + help=( + "path to a custom Overpass query file; use {{bbox}} as a " + "placeholder for the bounding box" + ), + ) parser.add_argument( "-i", "--input",