From 6bc12562378800738165ce4b902061156d7a66a3 Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Sun, 26 Apr 2026 10:12:16 +0100 Subject: [PATCH 1/8] Put map div higher in HTML tree Fixes the tile packing page --- website/public/.assetsignore | 3 +++ website/src/Home.svelte | 8 -------- website/src/Layout.svelte | 10 ++++++++++ website/src/Packing.svelte | 7 ++----- 4 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 website/public/.assetsignore diff --git a/website/public/.assetsignore b/website/public/.assetsignore new file mode 100644 index 0000000..abade04 --- /dev/null +++ b/website/public/.assetsignore @@ -0,0 +1,3 @@ +longest_lines/ +*.pmtiles + diff --git a/website/src/Home.svelte b/website/src/Home.svelte index 51c0406..fd9dd12 100644 --- a/website/src/Home.svelte +++ b/website/src/Home.svelte @@ -124,8 +124,6 @@ }); -
- @@ -163,12 +161,6 @@ diff --git a/website/src/Packing.svelte b/website/src/Packing.svelte index 62c7d99..71cc27d 100644 --- a/website/src/Packing.svelte +++ b/website/src/Packing.svelte @@ -46,14 +46,14 @@ } const tilesCSV = await result.text(); - state.map?.addSource('my-geojson', { + state.map?.addSource('packing', { type: 'geojson', data: tilesCSVToGeoJSON(tilesCSV), }); state.map?.addLayer({ id: 'fills', type: 'fill', - source: 'my-geojson', + source: 'packing', paint: { 'fill-outline-color': '#ff1234', 'fill-color': '#ff0000', @@ -168,6 +168,3 @@ - - From 71018932b85c4157360bcaefb221039a2356653d Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Sun, 26 Apr 2026 11:30:34 +0100 Subject: [PATCH 2/8] Support variably scaled .tiffs Depends on https://github.com/AllTheLines/CacheTVS/pull/56 --- README.md | 15 ++- crates/shared/src/projector.rs | 2 +- crates/tasks/src/atlas/daemon.rs | 9 +- .../tasks/src/atlas/longest_lines/grided.rs | 4 +- crates/tasks/src/atlas/longest_lines/index.rs | 5 +- .../tasks/src/atlas/longest_lines/overview.rs | 6 +- crates/tasks/src/atlas/run.rs | 3 +- crates/tasks/src/atlas/stitch_all.rs | 28 ++++-- crates/tasks/src/atlas/tile_job.rs | 69 ++++---------- crates/tasks/src/config.rs | 28 +++++- crates/tasks/src/packer.rs | 8 +- crates/tasks/src/stitch.rs | 92 ++++++++++++------- crates/tasks/src/tile.rs | 2 +- scripts/prepare_for_cloud.bash | 67 -------------- website/src/lib/getLongestLine.ts | 7 +- website/src/lib/utils.ts | 7 ++ 16 files changed, 163 insertions(+), 189 deletions(-) delete mode 100644 scripts/prepare_for_cloud.bash diff --git a/README.md b/README.md index 747107f..ddc6355 100644 --- a/README.md +++ b/README.md @@ -69,12 +69,23 @@ So can we strike an optimal balance? This is what the `Packer` in this repo trie Creates arbitrary tiles out of the global DEM data. +Stitch one: ``` cargo run --bin tasks -- stitch \ --dems /publicish/dems \ --centre -3.049208402633667,53.24937438964844 \ --width 366610.1875 ``` +(optionally takes `--scale` argument) + +Stitch all from a folder: +``` +cargo run --bin tasks -- atlas stitch-all \ + --tmp-directory /mnt/disks/data/tmp \ + --dems /mnt/disks/data/srtm/ \ + --master website/public/tiles.csv +``` +(optionally takes `--scale` argument) ## Calculate Total Viewsheds @@ -85,10 +96,6 @@ Outputs `.bt` heatmap. ## Prepare For Cloud ``` -# Note that all these depend on a `./output` path existing. -./ctl.sh prepare_for_cloud ../total-viewsheds/output/total_surfaces.bt -./ctl.sh prepare_for_cloud ../total-viewsheds/output/longest_lines.bt - ./ctl.sh make_pmtiles latest website/public/world.pmtiles ``` diff --git a/crates/shared/src/projector.rs b/crates/shared/src/projector.rs index b28d354..0298f89 100644 --- a/crates/shared/src/projector.rs +++ b/crates/shared/src/projector.rs @@ -3,7 +3,7 @@ use color_eyre::{Result, eyre::ContextCompat as _}; /// The radius of the planet in kilometers. -pub const EARTH_RADIUS: f32 = 6371.0; +pub const EARTH_RADIUS: f32 = 6378.137; /// A longtitude/latitude coordinate. #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Default)] diff --git a/crates/tasks/src/atlas/daemon.rs b/crates/tasks/src/atlas/daemon.rs index 6e3367f..36fef01 100644 --- a/crates/tasks/src/atlas/daemon.rs +++ b/crates/tasks/src/atlas/daemon.rs @@ -7,8 +7,7 @@ use apalis::{layers::WorkerBuilderExt as _, prelude::Layer as _}; /// The URL for the worker web UI. const WEB_UI_HOST: &str = "localhost:3003"; - -/// Start the Atlas daemon +/// Start the Atlas daemon. pub async fn start_all( _: &crate::config::Worker, broadcaster: std::sync::Arc>, @@ -53,10 +52,8 @@ async fn start_existing_machines() -> Result<()> { ); for job in jobs { - crate::atlas::machines::new_machine_job::new_machine_handler( - job.machine.as_ref().clone(), - ) - .await?; + crate::atlas::machines::new_machine_job::new_machine_handler(job.machine.as_ref().clone()) + .await?; } Ok(()) diff --git a/crates/tasks/src/atlas/longest_lines/grided.rs b/crates/tasks/src/atlas/longest_lines/grided.rs index ea85c4c..6c6f9a2 100644 --- a/crates/tasks/src/atlas/longest_lines/grided.rs +++ b/crates/tasks/src/atlas/longest_lines/grided.rs @@ -8,9 +8,9 @@ use tokio::io::AsyncWriteExt as _; #[repr(C)] #[derive(Copy, Clone, Debug, bytemuck::Zeroable, bytemuck::Pod)] pub struct Grided { - /// Longtitude + /// Longtitude. pub lon: f32, - /// Latitude + /// Latitude. pub lat: f32, /// The distance of the line of sight in meters. pub distance: u32, diff --git a/crates/tasks/src/atlas/longest_lines/index.rs b/crates/tasks/src/atlas/longest_lines/index.rs index a5da9e4..115e31b 100644 --- a/crates/tasks/src/atlas/longest_lines/index.rs +++ b/crates/tasks/src/atlas/longest_lines/index.rs @@ -25,10 +25,7 @@ pub async fn compile() -> Result<()> { let mut index = Vec::new(); for tile in tiles { let filename = tile.cog_filename(); - let line = format!( - "{filename} {}", - tile.width * crate::atlas::tile_job::DEM_SCALE - ); + let line = format!("{filename} {}", tile.width); index.push(line); } diff --git a/crates/tasks/src/atlas/longest_lines/overview.rs b/crates/tasks/src/atlas/longest_lines/overview.rs index f1c4b1b..35b897d 100644 --- a/crates/tasks/src/atlas/longest_lines/overview.rs +++ b/crates/tasks/src/atlas/longest_lines/overview.rs @@ -30,11 +30,11 @@ type StateHash = Arc>; /// Representation of a longest lines COG file. struct Tile { - /// The AEQD to lon/lat coordinate converter + /// The AEQD to lon/lat coordinate converter. projector: shared::projector::Convert, - /// The raw point data for the tile + /// The raw point data for the tile. buffer: gdal::raster::Buffer, - /// The width of the tile in points + /// The width of the tile in points. width: usize, /// Offset to the centre of the tile in points. offset: f64, diff --git a/crates/tasks/src/atlas/run.rs b/crates/tasks/src/atlas/run.rs index 94620e6..509226f 100644 --- a/crates/tasks/src/atlas/run.rs +++ b/crates/tasks/src/atlas/run.rs @@ -4,10 +4,9 @@ //! 2. Get tiles around a point that haven't been crunched yet //! 1. Stitch the relevant tile //! 2. TVS the tile -//! 3. Run `.ctl.sh prepare_for_cloud` for `total_surfaces.bt` and `longest_lines.bt`. //! //! These can and probably should be run manually after a bunch of tiles have been processed. -//! 4. Create the new `world.pmtile`: `./ctl.sh make_pmtiles latest website/public/world.pmtiles` +//! 3. Create the new `world.pmtile`: `./ctl.sh make_pmtiles latest website/public/world.pmtiles`. use color_eyre::{Result, eyre::ContextCompat as _}; diff --git a/crates/tasks/src/atlas/stitch_all.rs b/crates/tasks/src/atlas/stitch_all.rs index 9a6186e..8540177 100644 --- a/crates/tasks/src/atlas/stitch_all.rs +++ b/crates/tasks/src/atlas/stitch_all.rs @@ -11,7 +11,7 @@ use color_eyre::Result; pub struct StitchJob { /// Config from the CLI. pub config: crate::config::StitchAll, - /// The tile to process + /// The tile to process. pub tile: crate::tile::Tile, } @@ -55,18 +55,34 @@ pub async fn process(job: StitchJob) -> Result<()> { let dems_path = job.config.dems.display().to_string(); let centre = format!("{},{}", job.tile.centre.0.x, job.tile.centre.0.y); let width = job.tile.width.to_string(); + let tmp_directory = job.config.tmp_directory.display().to_string(); + let mut args = [ + "stitch", + "--dems", + &dems_path, + "--centre", + ¢re, + "--width", + &width, + "--output-dir", + &tmp_directory, + ] + .iter() + .map(std::string::ToString::to_string) + .collect::>(); + if let Some(scale) = job.config.scale { + args.extend(vec!["--scale".to_owned(), scale.to_string()]); + } let command = super::machines::connection::Command { - executable: "target/debug/tasks".into(), - args: vec![ - "stitch", "--dems", &dems_path, "--centre", ¢re, "--width", &width, - ], + executable: std::env::current_exe()?, + args: args.iter().map(std::string::String::as_str).collect(), ..Default::default() }; super::machines::local::Machine::command(command).await?; let stitch_tile_path = crate::stitch::canonical_filename(job.tile.centre.0.x, job.tile.centre.0.y); - let source = format!("output/{stitch_tile_path}"); + let source = format!("{tmp_directory}/{stitch_tile_path}"); let destination = format!("s3://viewview/stitched/{stitch_tile_path}"); let local = super::machines::local::Machine::connection(); local.sync_file_to_s3(&source, &destination).await?; diff --git a/crates/tasks/src/atlas/tile_job.rs b/crates/tasks/src/atlas/tile_job.rs index f839f38..8368aed 100644 --- a/crates/tasks/src/atlas/tile_job.rs +++ b/crates/tasks/src/atlas/tile_job.rs @@ -2,16 +2,13 @@ use crate::atlas::machines::connection::Connection; use crate::config::RUN_ID_LOCAL; +use apalis::prelude::WorkerContext; use clap::ValueEnum as _; use color_eyre::{Result, eyre::ContextCompat as _}; use std::sync::Arc; use std::time::SystemTime; -use apalis::prelude::WorkerContext; use tokio::sync::Mutex; -/// The size of each point in the raw DEM data. -pub const DEM_SCALE: f32 = 100.0; - /// The directory where all our viewview input/output goes. pub const WORKING_DIRECTORY: &str = "work"; @@ -23,10 +20,11 @@ pub const LONGEST_LINES_DIRECTORY: &str = "longest_lines"; pub struct TileJob { /// Config from the CLI. pub config: crate::config::Atlas, - /// The tile to process + /// The tile to process. pub tile: crate::tile::Tile, } +/// Data for running a single tile job. pub struct TileRunner<'mutex> { /// `mutex` is a borrowed mutex from the `Worker`. It makes sure that /// only a single `TileRunner` is running an L.o.S calculation. @@ -34,31 +32,30 @@ pub struct TileRunner<'mutex> { /// Details about this particular job. job: TileJob, /// The unique-per-job path prefix that is used to store files - /// in order to allow for unlimited concurrency + /// in order to allow for unlimited concurrency. job_directory: String, /// The connection to the machine where we run the compute parts of the job. machine: Arc, } -/// `TileState` is the necessary state that a `TileJob` worker needs to coordinate -/// between other workers. +/// Necessary state that a `TileJob` worker needs to coordinate between other workers. pub struct TileWorkerState { /// `mutex` makes sure that only one line of sight computation is happening at any given time. /// Any other part of the job can run at the same time, but running L.o.S is too computationally /// intensive to share - /// TODO: this might not have to be a mutex, but it is too time consuming to restart the jobs + /// TODO: this might not have to be a mutex, but it is too time consuming to restart the jobs. pub mutex: Arc>, - /// `daemon` holds an open ssh connection to the machine that all commands will be running on + /// `daemon` holds an open ssh connection to the machine that all commands will be running on. pub daemon: Arc, } /// `process_tile` does the work of processing a single tile. /// -/// It is a wrapper for `TileRunner` +/// It is a wrapper for `TileRunner`. pub async fn process_tile( job: TileJob, state: apalis::prelude::Data>, - ctx: WorkerContext + ctx: WorkerContext, ) -> Result<()> { tracing::info!("Processing tile: {:?}", job.tile); @@ -83,10 +80,10 @@ pub async fn process_tile( }; let result = runner.run().await; - if let Err(_) = result { + if result.is_err() { tracing::info!("shutting down worker {}", ctx.name()); ctx.stop()?; - return result + return result; } if cleanup { @@ -98,11 +95,11 @@ pub async fn process_tile( impl TileRunner<'_> { /// `run` sets up all directories, downloads necessary files, computes a L.o.S - /// and then does post-processing and uploads + /// and then does post-processing and uploads. async fn run(&self) -> Result<()> { self.ensure_directories().await?; - let bt_filepath = self.download_bt_file().await?; + let bt_filepath = self.download_stitched_geotiff().await?; self.compute(&bt_filepath).await?; @@ -143,11 +140,11 @@ impl TileRunner<'_> { // TODO: Make this its own job so it can be parallelised. /// Download the packer-found, pre-stitched `.bt` DEM tile data. - async fn download_bt_file(&self) -> Result { - let bt_filename = + async fn download_stitched_geotiff(&self) -> Result { + let filename = crate::stitch::canonical_filename(self.job.tile.centre.0.x, self.job.tile.centre.0.y); - let from = format!("s3://viewview/stitched/{bt_filename}"); - let to = format!("{}/{bt_filename}", self.job_directory); + let from = format!("s3://viewview/stitched/{filename}"); + let to = format!("{}/{filename}", self.job_directory); self.machine.sync_file_from_s3(&from, &to).await?; Ok(to) @@ -158,7 +155,6 @@ impl TileRunner<'_> { let _token = self.mutex.lock().await; let threads_as_string; - let scale = format!("{DEM_SCALE}"); let backend = self .job .config @@ -171,8 +167,6 @@ impl TileRunner<'_> { &bt_filepath, "--output-dir", &self.job_directory, - "--scale", - &scale, "--disable-image-render", "--backend", backend.get_name(), @@ -202,22 +196,10 @@ impl TileRunner<'_> { /// Process the assets needed to display the output on the website. async fn assets(&self) -> Result<()> { - self.prepare_for_cloud( - format!("{}/total_surfaces.bt", self.job_directory).as_str(), - &self.job.tile.cog_filename(), - ) - .await?; - if !self.job.config.is_local_run() { self.s3_put_raw_tvs_tiff().await?; } - self.prepare_for_cloud( - format!("{}/longest_lines.bt", self.job_directory).as_str(), - &self.job.tile.cog_filename(), - ) - .await?; - if !self.job.config.is_local_run() { self.s3_put_longest_lines_cog(&self.job.tile.cog_filename()) .await?; @@ -226,22 +208,6 @@ impl TileRunner<'_> { Ok(()) } - /// Prepare a processed tile for the website UI. - async fn prepare_for_cloud(&self, input: &str, output: &str) -> Result<()> { - let arguments = vec!["prepare_for_cloud", input, output]; - - self.machine - .command(crate::atlas::machines::connection::Command { - executable: "./ctl.sh".into(), - args: arguments, - env: vec![("OUTPUT_DIR", &self.job_directory)], - ..Default::default() - }) - .await?; - - Ok(()) - } - /// Sync the raw, pre-projected finished heatmap for the tile to our S3 bucket. /// /// There isn't a huge difference between this and the post-processed one, but it's a shame @@ -283,4 +249,3 @@ impl TileRunner<'_> { Ok(()) } } - diff --git a/crates/tasks/src/config.rs b/crates/tasks/src/config.rs index 5279190..8e3e4dd 100644 --- a/crates/tasks/src/config.rs +++ b/crates/tasks/src/config.rs @@ -47,7 +47,7 @@ pub enum AtlasCommands { LongestLinesOverviews(LongestLinesOverviews), /// Output the current run's config. CurrentRunConfig(CurrentRunConfig), - /// Stitch the entire world's `.bt` files and save them to S3. + /// Stitch the entire world's `.tiff` files and save them to S3. StitchAll(StitchAll), } @@ -100,6 +100,19 @@ pub struct Stitch { /// The width of the tile in meters. #[arg(long, value_name = "Tile width")] pub width: f32, + + /// Where to save stitched Geotiffs. + #[arg( + long, + value_name = "Path to output directory", + default_value = "./output" + )] + pub output_dir: std::path::PathBuf, + + /// Override dynamic scaling. If not supplied the scale is based on the tile's distance from + /// the equator. + #[arg(long, value_name = "Raster scale")] + pub scale: Option, } /// `cargo run atlas stitch-all` arguments. @@ -113,9 +126,22 @@ pub struct StitchAll { #[arg(long, value_name = "Path to master tiles list")] pub master: std::path::PathBuf, + /// Where to save stitched Geotiffs before uploading to the bucket. + #[arg( + long, + value_name = "Path to temporary directory", + default_value = "./output" + )] + pub tmp_directory: std::path::PathBuf, + /// Number of CPUS to use. #[arg(long, value_name = "Number of cpus", default_value_t = number_of_cpus_on_machine())] pub num_cpus: usize, + + /// Override dynamic scaling. If not supplied the scale is based on the tile's distance from + /// the equator. + #[arg(long, value_name = "Raster scale")] + pub scale: Option, } /// Worker daemon to run Atlas jobs. diff --git a/crates/tasks/src/packer.rs b/crates/tasks/src/packer.rs index 270ac1f..d10a7cb 100644 --- a/crates/tasks/src/packer.rs +++ b/crates/tasks/src/packer.rs @@ -994,9 +994,9 @@ mod test { #[test] fn minimum_tile_size_for_elevation() { assert_eq!(Packer::minimum_tile_size_for_elevation(0), 36_000.0); - assert_eq!(Packer::minimum_tile_size_for_elevation(1000), 225_769.8); - assert_eq!(Packer::minimum_tile_size_for_elevation(3000), 391_075.44); - assert_eq!(Packer::minimum_tile_size_for_elevation(8000), 638_748.75); - assert_eq!(Packer::minimum_tile_size_for_elevation(8848), 671_772.3); + assert_eq!(Packer::minimum_tile_size_for_elevation(1000), 225_896.2); + assert_eq!(Packer::minimum_tile_size_for_elevation(3000), 391_294.38); + assert_eq!(Packer::minimum_tile_size_for_elevation(8000), 639_106.2); + assert_eq!(Packer::minimum_tile_size_for_elevation(8848), 672_148.2); } } diff --git a/crates/tasks/src/stitch.rs b/crates/tasks/src/stitch.rs index 06af1f5..d293fca 100644 --- a/crates/tasks/src/stitch.rs +++ b/crates/tasks/src/stitch.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use color_eyre::{Result, eyre::ContextCompat as _}; +use shared::projector::EARTH_RADIUS; /// A virtual DEM that represents _all_ the DEM data for the planet. const VIRTUAL_DEM_FILE: &str = "index.vrt"; @@ -12,6 +13,9 @@ const VIRTUAL_DEM_FILE: &str = "index.vrt"; /// How we mark points as containing no data. const NODATA_VALUE: &str = "-32768"; +/// The dimensions of an SRTM3 pixel at the equator. Pixels get smaller the further from the equator. +const SRTM3_RESOLUTION_AT_EQUATOR: f64 = 92.766_242_327_727_98f64; + /// Entrypoint. pub async fn make_tile( machine: &Arc, @@ -19,7 +23,6 @@ pub async fn make_tile( ) -> Result { build_virtual_dem(machine, config).await?; let filename = stitch(machine, config).await?; - set_centre_as_extent(machine, config, &filename).await?; Ok(filename) } @@ -88,7 +91,7 @@ fn find_all_hgts(config: &crate::config::Stitch) -> Result> { /// The canonical name for the stitched file. It's needed to be able to put and get the file from /// the S3 bucket. pub fn canonical_filename(lon: f64, lat: f64) -> String { - format!("{lon},{lat}.bt") + format!("{lon},{lat}.tiff") } /// Call `gdalwarp` to construct a new stitched tile. Data will also be interpolated to metric. @@ -96,14 +99,16 @@ async fn stitch( machine: &Arc, config: &crate::config::Stitch, ) -> Result { - let resolution = 100.0; - let resolution_string = resolution.to_string(); + let resolution = get_resolution(config); + let resolution_string = format!("{resolution:.12}"); + let aeqd = format!( "+proj=aeqd +lat_0={} +lon_0={} +units=m +datum=WGS84 +no_defs", config.centre.1, config.centre.0 ); let output = format!( - "./output/{}", + "{}/{}", + config.output_dir.display(), canonical_filename(config.centre.0, config.centre.1) ); let hgt_index = config.dems.join(VIRTUAL_DEM_FILE).display().to_string(); @@ -120,8 +125,8 @@ async fn stitch( (half_width * 2.0) / 3.0 ); - let min = format!("-{half_width}"); - let max = format!("{half_width}"); + let min = format!("-{half_width:.12}"); + let max = format!("{half_width:.12}"); let arguments = vec![ "-overwrite", "-dstnodata", @@ -138,8 +143,14 @@ async fn stitch( &resolution_string, "-r", "bilinear", + "-co", + "COMPRESS=ZSTD", + "-co", + "PREDICTOR=2", + "-co", + "TILED=YES", "-of", - "BT", + "GTiff", &hgt_index, output.as_str(), ]; @@ -151,33 +162,50 @@ async fn stitch( }) .await?; + tracing::info!("Stitched tile saved to: {output}"); + Ok(output) } -/// Re-purpose the new tile's extent header to instead define its centre. -async fn set_centre_as_extent( - machine: &Arc, - config: &crate::config::Stitch, - file: &str, -) -> Result<()> { - let lon = config.centre.0.to_string(); - let lat = config.centre.1.to_string(); - let arguments = vec![ - "-a_ullr", - lon.as_str(), - lat.as_str(), - lon.as_str(), - lat.as_str(), - file, - ]; +/// Decide on the resolution to use for the stitched tile. +fn get_resolution(config: &crate::config::Stitch) -> f32 { + if let Some(resolution) = config.scale { + return resolution; + } + let vertical_resolution = calculate_longitude_resolution(config.centre.1); + #[expect( + clippy::as_conversions, + clippy::cast_possible_truncation, + reason = "This doesn't need accuracy" + )] + let resolution = (vertical_resolution.midpoint(SRTM3_RESOLUTION_AT_EQUATOR)) as f32; + resolution +} - machine - .command(crate::atlas::machines::connection::Command { - executable: "gdal_edit.py".into(), - args: arguments, - ..Default::default() - }) - .await?; +/// Calculate the vertical height of a pixel in the SRTM3 dataset. Because SRTM is degree-based +/// then the height of a pixel changes depending on how far it is from the equator. +fn calculate_longitude_resolution(latitude_degrees: f64) -> f64 { + const ARC_SECONDS: f64 = 3.0; // For SRTM3 + const SECONDS_PER_DEGREE: f64 = 3600.0; + let earth_radius_meters = f64::from(EARTH_RADIUS * 1000.0); + let latitude_radians = latitude_degrees.to_radians(); + let arc_second_radians = (ARC_SECONDS / SECONDS_PER_DEGREE).to_radians(); - Ok(()) + earth_radius_meters * latitude_radians.cos() * arc_second_radians +} + +#[expect(clippy::float_cmp, reason = "Just tests")] +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn pixel_height_at_latitude() { + assert_eq!( + calculate_longitude_resolution(0.0), + SRTM3_RESOLUTION_AT_EQUATOR + ); + assert_eq!(calculate_longitude_resolution(45.0), 65.595639015131f64); + assert_eq!(calculate_longitude_resolution(-45.0), 65.595639015131f64); + } } diff --git a/crates/tasks/src/tile.rs b/crates/tasks/src/tile.rs index b4606ed..9dd9998 100644 --- a/crates/tasks/src/tile.rs +++ b/crates/tasks/src/tile.rs @@ -11,7 +11,7 @@ use shared::projector::LonLatCoord; pub struct Tile { /// The centre of the tile. pub centre: shared::projector::LonLatCoord, - /// The width of the tile. Therefore, not the distance from the centre to an edge. Width is + /// The width of the tile in meters. Therefore, not the distance from the centre to an edge. Width is /// better than radius because it defines the minimum line of sight distance we are interested /// in. pub width: f32, diff --git a/scripts/prepare_for_cloud.bash b/scripts/prepare_for_cloud.bash deleted file mode 100644 index 186deb3..0000000 --- a/scripts/prepare_for_cloud.bash +++ /dev/null @@ -1,67 +0,0 @@ -function prepare_for_cloud { - set -e - set -x - - # Input `.bt` tile - local input=$1 - # Output `.tiff` tile - local output=$2 - - ensure_tiles_env - - # Lon/lat of the input tile - longitude=$(get_tiff_longitude "$input") - latitude=$(get_tiff_latitude "$input") - # Pixel resolution of the input tile - pixel_width=$(gdalinfo -json "$input" | jq '.size[0]') - # Width of the input tile - width=$((pixel_width * 50)) - - if [ -z "${output:-}" ]; then - output="$longitude"_"$latitude".tiff - echo "⚠️ Using tile's internal lon/lat for filename." - echo "⚠️ This may not match the coordinates of the Packer-defined tile." - fi - - # Where to keep the longest lines COG files. - LONGEST_LINES_DIR=${LONGEST_LINES_DIR:-$OUTPUT_DIR/longest_lines} - - # Just the filename without its path - base=$(basename "$input") - # The filename without its extension - stem="${base%.*}" - - mkdir -p "$LONGEST_LINES_DIR" - mkdir -p "$TMP_DIR" - rm -f "$TMP_DIR/"* - - plain_tif=$TMP_DIR/plain.tif - - # Convert to GeoTiff. - gdal_translate \ - -of GTiff \ - -a_nodata 0 \ - -co COMPRESS=DEFLATE \ - "$input" "$plain_tif" - - # The `.bt` format only has minimal support for georeferencing, so here we edit - # the GeoTiff's projection and extent. This is merely updating header metadata, - # it's not actually interpolating or anything like that. - gdal_edit.py \ - -a_ullr "-$width" "$width" "$width" "-$width" \ - -a_srs "+proj=aeqd +lat_0=$latitude +lon_0=$longitude +datum=WGS84" \ - "$plain_tif" - - if [[ $stem == "longest_lines" ]]; then - # Create the longest line of sight COG. It is used as-is by the website. - cog=$LONGEST_LINES_DIR/"$output" - gdal_translate \ - -of COG \ - -co BLOCKSIZE=32 \ - -co RESAMPLING=NEAREST \ - -co OVERVIEWS=NONE \ - -co COMPRESS=DEFLATE \ - -co PREDICTOR=3 \ - "$plain_tif" "$cog" - fi -} diff --git a/website/src/lib/getLongestLine.ts b/website/src/lib/getLongestLine.ts index 7fb0983..d0d9c87 100644 --- a/website/src/lib/getLongestLine.ts +++ b/website/src/lib/getLongestLine.ts @@ -9,6 +9,7 @@ import { clamp, endLoadingSpinner, getElevationAtCoord, + getScaleFromCog, Log, startLoadingSpinner, VERSION, @@ -245,8 +246,7 @@ async function findNearestCOGURLs(coordinate: LngLat) { const cogFilenames = []; for (const [filename, cog] of cogsIndex) { const distance = coordinate.distanceTo(cog.centre); - const scale = 100; // TODO: I thought we decided to set this in the indexer? - const radius = cog.width / 2 / scale; + const radius = cog.width / 2; Log.debug( `👀 Checking Longest Line COG: ${filename}`, `with centre: ${cog.centre} and radius ${radius}`, @@ -338,8 +338,7 @@ export function convertRasterXYToLngLat( geo.ProjCenterLongGeoKey, geo.ProjCenterLatGeoKey, ); - const resolution = cog.getResolution(); - const scale = Math.abs(resolution[0]); + const scale = getScaleFromCog(cog); const maxIndex = cog.getWidth() - 1.0; const offset = maxIndex / 2.0; const y_flipped = maxIndex - y; diff --git a/website/src/lib/utils.ts b/website/src/lib/utils.ts index 0fbdee6..0bbb0d5 100644 --- a/website/src/lib/utils.ts +++ b/website/src/lib/utils.ts @@ -1,4 +1,5 @@ import nprogress from 'accessible-nprogress'; +import type { GeoTIFFImage } from 'geotiff'; import { LngLat, LngLatBounds } from 'maplibre-gl'; import type { AppState } from '../state.svelte'; @@ -202,3 +203,9 @@ export async function getElevationAtCoord(coord: LngLat) { const payload = await result.json(); return payload.geoPoints[0].elevation; } + +export function getScaleFromCog(cog: GeoTIFFImage) { + const resolution = cog.getResolution(); + const scale = Math.abs(resolution[0]); + return scale; +} From 16af5d97e022977ae8aa2abec1625f8132c3af6c Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Tue, 28 Apr 2026 17:04:00 +0100 Subject: [PATCH 3/8] Add `cargo clippy` to lint CI job --- .github/workflows/ci.yaml | 16 ++++++++++++++ crates/shared/src/lib.rs | 1 - crates/tasks/src/atlas/machines/connection.rs | 2 +- .../tasks/src/atlas/machines/digital_ocean.rs | 2 +- .../tasks/src/atlas/machines/google_cloud.rs | 22 +++++++++++-------- .../src/atlas/machines/new_machine_job.rs | 6 ++--- crates/tasks/src/atlas/machines/vultr.rs | 2 +- crates/tasks/src/atlas/machines/worker.rs | 4 ++-- crates/tasks/src/atlas/run.rs | 5 +++++ 9 files changed, 41 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e8c8d18..0b11ae5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,17 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Linux system dependencies + run: | + sudo apt-get update -y + sudo apt-get install libgdal-dev gdal-bin + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.94.1 + components: clippy,rustfmt - name: Install pnpm uses: pnpm/action-setup@v4 with: @@ -60,3 +71,8 @@ jobs: working-directory: website run: node_modules/.bin/biome check + - name: Lint + run: cargo clippy -- --deny warnings + - name: Formatting + run: cargo fmt --check + diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 5daf4c0..dd8a645 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -8,4 +8,3 @@ pub mod metadata; pub mod projector; pub mod utils; - diff --git a/crates/tasks/src/atlas/machines/connection.rs b/crates/tasks/src/atlas/machines/connection.rs index 50c8e45..40256a2 100644 --- a/crates/tasks/src/atlas/machines/connection.rs +++ b/crates/tasks/src/atlas/machines/connection.rs @@ -20,7 +20,7 @@ pub struct Command<'command> { pub args: Vec<&'command str>, /// Environment variables to pass to the command process. pub env: Vec<(&'command str, &'command str)>, - /// The directory in which to run the command + /// The directory in which to run the command. pub current_dir: Option, } diff --git a/crates/tasks/src/atlas/machines/digital_ocean.rs b/crates/tasks/src/atlas/machines/digital_ocean.rs index 3fdefe9..a374524 100644 --- a/crates/tasks/src/atlas/machines/digital_ocean.rs +++ b/crates/tasks/src/atlas/machines/digital_ocean.rs @@ -1,4 +1,4 @@ -//! Managing Digital Ocean resources +//! Managing Digital Ocean resources. use color_eyre::Result; diff --git a/crates/tasks/src/atlas/machines/google_cloud.rs b/crates/tasks/src/atlas/machines/google_cloud.rs index fdc33fa..0f721e7 100644 --- a/crates/tasks/src/atlas/machines/google_cloud.rs +++ b/crates/tasks/src/atlas/machines/google_cloud.rs @@ -1,16 +1,15 @@ -//! `google_cloud` is a Google Cloud +//! `google_cloud` is a Google Cloud. use color_eyre::eyre::Result; -use std::net::{IpAddr}; -use std::time::{SystemTime}; +use std::net::IpAddr; +use std::time::SystemTime; - -/// Machine implements the `machine::Machine` trait for privisioning a Google Cloud machine +/// Machine implements the `machine::Machine` trait for privisioning a Google Cloud machine. pub struct Machine; impl Machine { /// provision runs the whole provisioning workflow for a Google Cloud machine, - /// returning the IP address of the new machine + /// returning the IP address of the new machine. async fn provision(name: &str, ssh_key_id: &str) -> Result { let command = super::connection::Command { executable: "./ctl.sh".into(), @@ -28,13 +27,18 @@ impl Machine { /// /// Google Cloud frequently hands out the same IP to different runs creating /// issues with host checking. Instead of infecting all other ssh commands with key checking, - /// it is much easier just to clear that IP out from `known_hosts` + /// it is much easier just to clear that IP out from `known_hosts`. async fn wait_for_machine_to_boot(ip_address: IpAddr) -> Result<()> { let ip_string = ip_address.to_string(); let delete_ip = super::connection::Command { executable: "ssh-keygen".into(), - args: vec!["-f", "/home/ryan/.ssh/known_hosts", "-R", ip_string.as_str()], + args: vec![ + "-f", + "/home/ryan/.ssh/known_hosts", + "-R", + ip_string.as_str(), + ], ..Default::default() }; super::local::Machine::command(delete_ip.clone()).await?; @@ -61,7 +65,7 @@ impl Machine { color_eyre::eyre::bail!("Timed out waiting for machine {ip_string} to boot."); } - /// `init` provisions a `GoogleCloud` Debian machine with the Ubuntu startup script + /// `init` provisions a `GoogleCloud` Debian machine with the Ubuntu startup script. async fn init(ip_address: IpAddr) -> Result { let connect = format!("atlas@{ip_address}"); diff --git a/crates/tasks/src/atlas/machines/new_machine_job.rs b/crates/tasks/src/atlas/machines/new_machine_job.rs index d73711b..37991d2 100644 --- a/crates/tasks/src/atlas/machines/new_machine_job.rs +++ b/crates/tasks/src/atlas/machines/new_machine_job.rs @@ -21,9 +21,7 @@ impl NewMachineJob { } /// The callback to run when a new machine job is added. -pub async fn new_machine_handler( - job: super::new_machine_job::NewMachineJob, -) -> Result<()> { +pub async fn new_machine_handler(job: super::new_machine_job::NewMachineJob) -> Result<()> { let machine_task_id = get_machine_task_id_hack(job.clone()).await?; tokio::spawn(async move { @@ -36,7 +34,7 @@ pub async fn new_machine_handler( } /// Get the internal ID of the job. Just a hack until this is fixed: -/// +/// . async fn get_machine_task_id_hack( job_to_find: super::new_machine_job::NewMachineJob, ) -> Result { diff --git a/crates/tasks/src/atlas/machines/vultr.rs b/crates/tasks/src/atlas/machines/vultr.rs index f7da7be..36543da 100644 --- a/crates/tasks/src/atlas/machines/vultr.rs +++ b/crates/tasks/src/atlas/machines/vultr.rs @@ -1,4 +1,4 @@ -//! Managing Vultr resources +//! Managing Vultr resources. use color_eyre::Result; diff --git a/crates/tasks/src/atlas/machines/worker.rs b/crates/tasks/src/atlas/machines/worker.rs index f8b9f87..af62c3c 100644 --- a/crates/tasks/src/atlas/machines/worker.rs +++ b/crates/tasks/src/atlas/machines/worker.rs @@ -62,7 +62,8 @@ pub async fn tile_processor( }; // clear out the previous run's state - state.daemon + state + .daemon .command(crate::atlas::machines::connection::Command { executable: "rm".into(), args: vec!["-rf", WORKING_DIRECTORY], @@ -74,7 +75,6 @@ pub async fn tile_processor( // Computation concurrency is effectively 1 due to mutex locking in the tile job. let worker_concurrency = 2; - WorkerBuilder::new(tile_worker_name.clone()) .backend(tile_store) .data(Arc::new(state)) diff --git a/crates/tasks/src/atlas/run.rs b/crates/tasks/src/atlas/run.rs index 509226f..ac2ab96 100644 --- a/crates/tasks/src/atlas/run.rs +++ b/crates/tasks/src/atlas/run.rs @@ -112,6 +112,11 @@ impl Atlas { tile: master_tile.data, }; + #[expect( + clippy::as_conversions, + clippy::cast_possible_truncation, + reason = "Truncation is unlikely to happen™" + )] // map the priority of the tile width (10,000->900,000) to an integer 1-9 // and then invert the priority so that the smallest go first let priority = -((tile_args.tile.width / 100_000.0f32) as i32); From e5f424868d1d12659608a059587481eaded7a1f0 Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Wed, 29 Apr 2026 15:33:14 +0100 Subject: [PATCH 4/8] Fixes against our latest sample compute run --- README.md | 4 ++-- scripts/make_pmtiles.bash | 1 + website/src/TileWorker.ts | 4 +++- website/src/lib/renderLongestLine.ts | 3 +-- website/src/lib/utils.ts | 5 +---- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ddc6355..dcaf538 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ bunch of tiles have been processed: # This requires a machine with a lot of disk, RAM and CPU. As of writing, I'd recommend # 3TB disk, >150GB RAM and at least 48 cores. # Replace `latest` with `local` to skip syncing files to S3. -./ctl.sh make_pmtiles latest work/world.pmtiles +./ctl.sh make_pmtiles latest work/world.pmtiles 100 ``` * Create an index of all the COG (optimised GeoTiff) files that contain all the longest lines of sight for every point on the planet. Should only take seconds to run. @@ -165,7 +165,7 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas longest-lines-index RUST_LOG=off,tasks=trace cargo run --release --bin tasks -- \ atlas longest-lines-overviews \ - --tiffs work/longest_lines + --tiffs work/longest_lines --run-id 0.1 ``` ## Static Site Website diff --git a/scripts/make_pmtiles.bash b/scripts/make_pmtiles.bash index 147f600..7671c7f 100644 --- a/scripts/make_pmtiles.bash +++ b/scripts/make_pmtiles.bash @@ -9,6 +9,7 @@ function make_pmtiles { # Output `.pmtile` tile. local output=$2 + # Resolution in metres of the output .pmtile local resolution=${3} ensure_tiles_env diff --git a/website/src/TileWorker.ts b/website/src/TileWorker.ts index 4cdd1cc..4f732e8 100644 --- a/website/src/TileWorker.ts +++ b/website/src/TileWorker.ts @@ -41,13 +41,15 @@ self.onmessage = async (event: MessageEvent) => { let bytes: Uint8Array | ArrayBuffer; let url = `${pmtilesSource}/${z}/${x}/${y}.bin${CACHE_BUSTER}`; - if (z <= 8) { + if (pmtilesSource.endsWith('world') && z <= 8) { // For zoom levels 0-8 we skip the Cloudflare Worker and use a CDN cache. This saves // on Cloudflare Worker monthly quotas. url = url .replace(`https://${MAP_SERVER_SUBDOMAIN}.`, 'https://cdn.') .replace('world.pmtiles/world', 'cache') .replace('.bin', ''); + } else { + console.warn('Not using CDN for tiles at low zoom levels'); } if (isProductionMapServer) { diff --git a/website/src/lib/renderLongestLine.ts b/website/src/lib/renderLongestLine.ts index a299f5f..54dfaa0 100644 --- a/website/src/lib/renderLongestLine.ts +++ b/website/src/lib/renderLongestLine.ts @@ -6,7 +6,6 @@ import { state } from '../state.svelte.ts'; import { getLongestLine, makeGoogleEarthLink } from './getLongestLine.ts'; import { - ANGLE_SHIFT, aeqdProjectionString, computeBBox, disablePointer, @@ -71,7 +70,7 @@ export async function render(lngLat: LngLat) { console.log(longestLine); } - longestLine.angle = longestLine.angle + ANGLE_SHIFT; + longestLine.angle = longestLine.angle - 90; const θ = toRadians(longestLine.angle); const dx = longestLine.distance * Math.cos(θ); diff --git a/website/src/lib/utils.ts b/website/src/lib/utils.ts index 0bbb0d5..0626893 100644 --- a/website/src/lib/utils.ts +++ b/website/src/lib/utils.ts @@ -12,13 +12,10 @@ export const PMTILES_SERVER = `${MAP_SERVER}/runs/${VERSION}/pmtiles/${WORLD_PMT // This is for busting Cloudflare asset cache. Like for an updated `world.pmtiles`, // longest lines index, etc. -export const CACHE_BUSTER = '?buster=23:05-19/04/2026'; +export const CACHE_BUSTER = '?buster=15:28-29/04/2026'; export const EARTH_RADIUS = 6371_000.0; -// Inherited from the TVS algorithm. It's to counter unfavourable floating point rounding. -export const ANGLE_SHIFT = 0.0001; - export const Log = { // biome-ignore lint/suspicious/noExplicitAny: needed for debugging. debug: (...data: any[]) => { From a7b324b9e279e290a00bed4625114e6849e1b269 Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Thu, 30 Apr 2026 18:40:22 +0100 Subject: [PATCH 5/8] Update for ryan-world2-run-refraction23 world run --- scripts/cache_tiles.bash | 4 ++-- website/src/TileWorker.ts | 2 +- website/src/lib/utils.ts | 6 +++--- website/src/lib/worldLines.ts | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/cache_tiles.bash b/scripts/cache_tiles.bash index d068ad0..e2a79a1 100644 --- a/scripts/cache_tiles.bash +++ b/scripts/cache_tiles.bash @@ -14,7 +14,7 @@ function cache_cacheable_tiles { export -f cache_tile # Takes about 15 minutes for zoom levels 0-8 - for z in {0..1}; do + for z in {0..8}; do max=$((2 ** z - 1)) parallel --jobs 20 cache_tile "$version $z {1} {2}" ::: $(seq 0 $max) ::: $(seq 0 $max) done @@ -32,7 +32,7 @@ function cache_tile { local x=$3 local y=$4 local path="$z/$x/$y" - local base="runs/$version/pmtiles/world.pmtiles/world" + local base="runs/$version/pmtiles/world" curl \ --create-dirs \ diff --git a/website/src/TileWorker.ts b/website/src/TileWorker.ts index 4f732e8..90717e8 100644 --- a/website/src/TileWorker.ts +++ b/website/src/TileWorker.ts @@ -46,7 +46,7 @@ self.onmessage = async (event: MessageEvent) => { // on Cloudflare Worker monthly quotas. url = url .replace(`https://${MAP_SERVER_SUBDOMAIN}.`, 'https://cdn.') - .replace('world.pmtiles/world', 'cache') + .replace('pmtiles/world', 'pmtiles/cache') .replace('.bin', ''); } else { console.warn('Not using CDN for tiles at low zoom levels'); diff --git a/website/src/lib/utils.ts b/website/src/lib/utils.ts index 0626893..f6e55d0 100644 --- a/website/src/lib/utils.ts +++ b/website/src/lib/utils.ts @@ -3,16 +3,16 @@ import type { GeoTIFFImage } from 'geotiff'; import { LngLat, LngLatBounds } from 'maplibre-gl'; import type { AppState } from '../state.svelte'; -export const VERSION = 'ryan-fullworld-raw'; +export const VERSION = 'ryan-world2-run-refraction23'; export const CDN_BUCKET = 'https://cdn.alltheviews.world'; export const MAP_SERVER_SUBDOMAIN = 'pmtiles'; export const MAP_SERVER = `https://${MAP_SERVER_SUBDOMAIN}.alltheviews.world`; -export const WORLD_PMTILES = 'world.pmtiles/world'; // TODO move the file to its proper place +export const WORLD_PMTILES = 'world'; export const PMTILES_SERVER = `${MAP_SERVER}/runs/${VERSION}/pmtiles/${WORLD_PMTILES}`; // This is for busting Cloudflare asset cache. Like for an updated `world.pmtiles`, // longest lines index, etc. -export const CACHE_BUSTER = '?buster=15:28-29/04/2026'; +export const CACHE_BUSTER = '?buster=18:36-30/04/2026'; export const EARTH_RADIUS = 6371_000.0; diff --git a/website/src/lib/worldLines.ts b/website/src/lib/worldLines.ts index 79b35f7..d15999a 100644 --- a/website/src/lib/worldLines.ts +++ b/website/src/lib/worldLines.ts @@ -18,7 +18,8 @@ import { } from './utils'; /// The filename for the longest lines grid. -const LONGEST_LINES_GRIDED_FILENAME = 'longest_lines_grided.bin'; +const LONGEST_LINES_GRIDED_FILENAME = + 'longest_lines_cogs/longest_lines_grided.bin'; export class LongestLineH3 { coordinate: LngLat; From 44a19bcdcd7b5a7fd3e653d48d43fe34e1947ce5 Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Sat, 2 May 2026 19:44:16 +0100 Subject: [PATCH 6/8] Compile longest lines H3 index using threads Previously we were using async tasks which are not at all favourable to CPU-bound work. --- README.md | 8 +- .../tasks/src/atlas/longest_lines/overview.rs | 125 ++++++++++-------- screenshot.webp | Bin 0 -> 34706 bytes 3 files changed, 79 insertions(+), 54 deletions(-) create mode 100644 screenshot.webp diff --git a/README.md b/README.md index dcaf538..256962a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # A View Of All Views +![Screenshot of app](screenshot.webp) + This repo is for all the supporting code used to find and display the longest line of sight on the planet. The main viewshed algorithm is another repo https://github.com/AllTheLines/CacheTVS @@ -91,7 +93,7 @@ cargo run --bin tasks -- atlas stitch-all \ Using https://github.com/AllTheLines/CacheTVS -Outputs `.bt` heatmap. +Outputs `total_surfaces.tiff` and `longest_lines.tiff`. ## Prepare For Cloud @@ -146,7 +148,7 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ Atlas doesn't run the following commands, you'll want to manually run them after a bunch of tiles have been processed: -* Create the gigantic (10s of GBs) global `.pmtile` that contains the TVS heatmap for the entire planet. +* Create the gigantic (~500GB) global `.pmtile` that contains the TVS heatmap for the entire planet. ``` # This requires a machine with a lot of disk, RAM and CPU. As of writing, I'd recommend # 3TB disk, >150GB RAM and at least 48 cores. @@ -154,6 +156,8 @@ bunch of tiles have been processed: ./ctl.sh make_pmtiles latest work/world.pmtiles 100 ``` +Note that this process creates a _temporary_ version of the `.pmtile`, and then copies that to its final destination. So you need at least 1.2TB of disk for a world PMTile. And then of course you need space to download the raw .tiffs, and space to save the post-processed versions of those .tiffs. So at least 2TB is a good idea. + * Create an index of all the COG (optimised GeoTiff) files that contain all the longest lines of sight for every point on the planet. Should only take seconds to run. ``` RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas longest-lines-index diff --git a/crates/tasks/src/atlas/longest_lines/overview.rs b/crates/tasks/src/atlas/longest_lines/overview.rs index 35b897d..918a988 100644 --- a/crates/tasks/src/atlas/longest_lines/overview.rs +++ b/crates/tasks/src/atlas/longest_lines/overview.rs @@ -23,10 +23,10 @@ struct LongestLine { /// A hash where each key is an H3 grid cell and its value is the longest line of sight within /// that grid cell. -type HashMap = std::collections::HashMap; +type LongestInGrids = std::collections::HashMap; -/// Keep track of the current longest line in each H3 grid. -type StateHash = Arc>; +/// Keep track of work per tile. +type StateHash = Arc>>; /// Representation of a longest lines COG file. struct Tile { @@ -42,7 +42,7 @@ struct Tile { impl Tile { /// Entrypoint. - fn process(path: &std::path::Path) -> Result { + fn process(path: &std::path::Path) -> Result { let centre = Self::parse_centre_coords(path)?; let buffer = Self::load(path)?; let width = buffer.width(); @@ -105,10 +105,10 @@ impl Tile { /// Iterate through every longest line in COG, associate it with a H3 grid, check to see if /// it's the longest in the grid and set record it if so. - fn find_longest_lines(&self) -> Result { + fn find_longest_lines(&self) -> Result { tracing::trace!("Looping through {} points", self.buffer.len()); - let mut local = HashMap::new(); + let mut local = LongestInGrids::new(); for (index, &value) in self.buffer.data().iter().enumerate() { let coord = self.index_to_coord(index); let lonlat = self.coord_to_lonlat(coord)?; @@ -192,45 +192,66 @@ pub async fn run(config: &crate::config::LongestLinesOverviews) -> Result<()> { let world_clone = Arc::clone(&world); let jobs_clone = Arc::clone(&jobs); let completed_jobs = Arc::clone(&completed_jobs_master); - let handle = tokio::task::spawn(async move { - tracing::debug!("Spawning worker {worker}"); - - loop { - let job = jobs_clone.write().await.pop(); - match job { - Some(tiff) => { - let result: Result<()> = async { - let local = Tile::process(&tiff)?; - update_state(&world_clone, local).await?; - completed_jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - tracing::info!( - "{}/{total_jobs} completed jobs", - completed_jobs.load(std::sync::atomic::Ordering::Relaxed) - ); - Ok(()) + + tracing::debug!("Spawning worker {worker}"); + let handle = std::thread::spawn(move || { + #[expect(clippy::expect_used, reason = "I'm lazy")] + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("Couldn't build Tokio runtime in current thread"); + + runtime.block_on(async { + loop { + let job = jobs_clone.write().await.pop(); + match job { + Some(tiff) => { + let filename = tiff.display().to_string(); + let result: Result<()> = async { + let local = Tile::process(&tiff)?; + world_clone.write().await.insert(filename.clone(), local); + let current_world = world_clone.read().await.clone(); + std::thread::spawn(move || compile_world(¤t_world)); + completed_jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + tracing::info!( + "{}/{total_jobs} completed jobs", + completed_jobs.load(std::sync::atomic::Ordering::Relaxed) + ); + Ok(()) + } + .await; + if let Err(error) = result { + tracing::error!( + "Error running longest lines job for {filename}: {error}" + ); + #[expect(clippy::exit, reason = "Everything needs to work!")] + std::process::exit(1); + } } - .await; - if let Err(error) = result { - tracing::error!("Error running longest lines job: {error}"); - #[expect(clippy::exit, reason = "Everything needs to work!")] - std::process::exit(1); + None => { + tracing::debug!("Worker {worker} shutting down"); + break; } } - None => { - tracing::debug!("Worker {worker} shutting down"); - break; - } } - } + }); }); handles.push(handle); } - for result in futures::future::join_all(handles).await { - result?; + for handle in handles { + #[expect(clippy::expect_used, reason = "I'm lazy")] + handle.join().expect("Error joining thread"); } + let grided = compile_world(&world.read().await.clone())?; + super::grided::write( + format!("./output/{LONGEST_LINES_GRIDED_FILENAME}",).into(), + &grided, + ) + .await?; + sync_to_s3( format!("./output/{LONGEST_LINES_GRIDED_FILENAME}").into(), &config.run_id, @@ -240,17 +261,22 @@ pub async fn run(config: &crate::config::LongestLinesOverviews) -> Result<()> { Ok(()) } -/// Take all the newly found longest lines in a COG and check to see if they're longer than any of -/// the currently recorded global ones. -async fn update_state(world: &StateHash, local: HashMap) -> Result<()> { +/// Take all the per-tile grided longest lines and compile a global canonical grid of longest lines. +fn compile_world( + world: &std::collections::HashMap, +) -> Result> { + let mut master = LongestInGrids::new(); + #[expect(clippy::iter_over_hash_type, reason = "We don't mind about ordering")] - for (cell, line) in local { - if let Some(existing) = world.write().await.get_mut(&cell) { - if line.packed.distance() > existing.packed.distance() { - *existing = line; + for grid_for_tile in world.values() { + for (cell, line) in grid_for_tile { + if let Some(existing) = master.get_mut(cell) { + if line.packed.distance() > existing.packed.distance() { + *existing = *line; + } + } else { + master.insert(*cell, *line); } - } else { - world.write().await.insert(cell, line); } } @@ -261,9 +287,10 @@ async fn update_state(world: &StateHash, local: HashMap) -> Result<()> { }; let mut grided = Vec::new(); - let count = world.read().await.len(); + let count = master.len(); + #[expect(clippy::iter_over_hash_type, reason = "We don't mind about ordering")] - for entry in world.read().await.values() { + for entry in master.values() { if entry.packed.distance() > longest.packed.distance() { longest = *entry; } @@ -296,13 +323,7 @@ async fn update_state(world: &StateHash, local: HashMap) -> Result<()> { ); }; - super::grided::write( - format!("./output/{LONGEST_LINES_GRIDED_FILENAME}",).into(), - &grided, - ) - .await?; - - Ok(()) + Ok(grided) } /// Find all the `.tiff`s in a given directory. diff --git a/screenshot.webp b/screenshot.webp new file mode 100644 index 0000000000000000000000000000000000000000..44be34705d34cb801effc2b151f7c0a116ec0cb8 GIT binary patch literal 34706 zcmV(=K-s@iNk&GBhX4RqMM6+kP&godhX4RjG6J0eDue?O0zNSqibNtIp%F=3upk2j zv$uW}3}Bo+U7%?JoXW(l$o^4(uSY5nT|hySbUC;B(}f7*|B zkM`fQe(-;+|F!+Q_yhih{zv~ewdejH|NiGcrE2>iGDXZ&x?56$1EZ_w_k-M`}c1^e0fU-^D1AMSd6 z`hWYM{2t(+(f_^w!~D(t|NA5VAB=adf86_k`vm@<{r~-6_@D6K_rJq^EB~$k&;7^d zFTS6#KllHx_y_)P{j2^z_|M)?^MC*U^ZoUEUHzQ)DE(>wk@@~7NR*bsxQ&TTP@h9_ z{P1JritNSd$K(lBpSHp=K!-;RO&*t7^oX4Ig=^l_O!|OZvVsOof5PVajB-`1A`I~FhL2?^`hx>O~IIG<%KqsoUReaBD+aaP-T>O$CN(NT z1~QE=d?XV$ouv5lApmWS4t6-Z7S^{3W``WuHIyj^oXEy+;@P?(7v545l{SBDqa&{D zxqCR7c**AIYvvYzruqVUwcO$Zhj~U+@N0fRtUWuf>SbYp1!&aLPO41!Lua_l%R9W+ znF!bkf(tsMH|rVcOHFJiotC2@^Dr(GDm;1 zW`H85Nr`&9f+zTu@68|(l58IlZf+x4TM_=rl%>4{G_fjF)>o<=M4{%Y)ISMM>8M9O zh`&$e##XlbFl-=fcF&gA_`5b_n1^hujuC51Dd`mbJJ{`Je7oB^I@P2=o;e%I+QkaL zUfd2&32H`^N$Bq-DzzGhJ(S?^c)V;Of5Jbz!1q{#Y*vp#)4fV)ehN;E&SBEB#8uxh z=!5R6HErtI?gyjCy^+$CLHxX67s%cCCiaKa7AZ=T$0&~Lp6Gq_Sy($8BN(GxN#lo6 zz4PLV>{d{2e|&CiWNeeS%&@86QWKF~| zKaM34PN-}q$!v4IPTetpZEnil8taD@wj<)cf*M$pd!hAWXRl~>Ob{O?8&Zxj2n(FP{ zzc*GjFq>+(gc_!GuD2Lp?CJov=n9?)qe-?A?$v5_1%I*486{0pCB=DOE>o?Z94-v? z$K|2^2`KX+qweme zcJ+o^k~x8~wcBU}5MQISD9I2`KFCa)m=~(3(V!uuQa^NAG_XonDenT7_YoUK!7Zhi z(hS!|y1(N{4@O&35(R|`>j9^&(Mi1PT4qeuBjcA71gA}IY31Iz35p8*oOcof%o^Z5jS&D6Z{od-c5ZTZ<4!>_XV6w?k=tZc(_c#8hIGOE zJgN2QX4MDIvmY}_JXCC((eldn=b{y9xJUlGOd#phf?;cu6VOX^Ypf)H^t1~$EnEoW zF!AI+Logy)zgAbv4)QtC7&$2Bt=6ukWc{}(gztcVbL=UyuMF9zef1D)EHB=wJ5jC{0Hq9mSioDlvoAH&fHS|dIKN5j!x38r zu>Q`Q2gxAsy_=ns?b*ZKVm{S82-`2xn};*Zu0*k^BW@2iY;IMTe~2X?yniq&%%?Mc zy=_LV^eDbT;8b7^J~e8&V%TRprzU~C_!8_RqLRFAl+*Zuf;&f2FSbaHIq(F`wEvO_ zBXljN{J4=Vf-CH z+xJve!M1Tv(xJgYhPunjY7jcJ^xju|!I2))PNLJFA?hX)^iv~e${C?codt4|!^r-N zo`a~WGt$Q{h&aq@RCLeG_Ehu|-5TnrWKKmCT+4>6&Lu@r=XV5+}%G@NN_rH zy_oo|CQ`W~KQbPCRkbdP1gGXt>7boDboN;!REhIDK6OR6L78u|7cXll&Q-c4nAKZL zkg-$Iu~TP4UY>YVU*k%if?K0p%E7aQpAgPUwhlQjBR`Mu#tR7*5WR1dY4mCZKBSt2 zeJpSV^f3Kaf#?cl7xqzad!Hi(r_F3}U+L(Gg0nQNwEi+IS*flMtOF{Zi~MO*&`We} zsA_E?0mTYLGuK69er6<74~-V%6JTbJB1W&`bEDI%fc-_C@FzlKmTmDxwbRXAnE<+; zd86zNnQUZC9-T)JP!~ChQumGQO;gZkMwxYg#+5w%hXFL}1)Ie-qRbDqjE~D-@sR(l z4n_XD2k>mAZw2mXUWJXul>T7igy=;gc2geGF|A=>izoaIJ{4$Co@(J0;adASq+tjb zMMp9XWZ*g&_Ehu`hesy~(||*)JYbb8nuupBX!yH>cV97?WDs(TKROPk6SH~^{tTPs zf}O#wf-oHG+CTj0JAgEpIB)<@hzy67Q@7#>ACj1U5o$<7N61^yOLSfpPYQkwLw9~` zWA?i?FehDeEF;1B8XW(gf5ly^$+g1|iY0Ta0R1hF3tXPI#LsOMAG=eqj?|2PyEe~S z${r9ITNjhDq%flP*N4q0pE~yrJq2zU41%Ks^=lZj1F|5W4H`g5CEO7-I#C&Ktv+tK zm7^TwgO%(#>+;!hH(*_79aGRthYeQC@6&$xWHn?r%hR9zv1yLFD^bI7l_E%od1VV& z59e+H;WXbnUI#aVSav(>&p!&ErV#&v%46yKH!n=6(|EtjbumJSNW6Nw81_{361XFb zN?-qpLXQmTE$UvK1~0nBgB}k359MhSNTz+lfiTF0dopz^ij2DoYeMzI#GC49AI3rq zQGUdZAVbY!acKU&MqvC@zA1Sn7ieGO)zHVPkKp_w_3=EG1=aF#6J%Xu^|zZ`_$Iu6 zoj>T%WhS-ME85I{@I*YHDF*`lt~a-m67)CH7j(dV_fG8xoohqTS?#3ZUrt*23RWG;3V*f{VjJM zv-1DIBRGvNxiR=Y3E`;pi+o~$#9{GJFwm%!s(yBeir33ys>e3WgFr=D&wq><*#?46 zvC>NLJ*~W#&>D?op%+?HM@aME_PST|Gl+8n4jLv?%x#x<~gi|v902|h9ruxuQRag0+X z457Y_$Q?q{=EXP|Mst`gR1?AgWGr8T7T7Q<~sa;sh6aDodqW#*hijuk-taLva8CD<(JiQo|g&V50Nrj@V| z(&}|2?HTVMl2L5BVjMj4o;jyc-tDyohp$f{@GOWx*E1t>UIYsHfC7e|Sp-cgn=s4z z!Du=nJFd~fF|Sv!D=~Y~h!H(9`k}nE`A7W4RwIRdbvV?_rR?NwP#l)4h^NSs0{8(Z zM9XLatZf_$)6Rm|19`0}vewW23c6DgCr?xq2qn6S_vwJX@-u(HZhwEaj(nP|D#X6 z>otkU6geG=F9UXQ{-q4VQG4B#CMnRsU<;RC9QWMVgzziWQVx%+&gre}T+=M+P75r_ zDC+F1PK&Y*`*2=P!aG2JBP9cY5}(`V?EqA|O6)SGY=v|WBa1niUTJIDn%%3aPG*4M zU?Sd=UE99$W35U)UYzmKb4}^2C-8n&^1|Ba*Hnp*G*X=D5UF!7o!6{ps?`ufX075O zLI@t?oBx>toe%b;r67+Rj6cAh%%LzxHXJSBt=`d*4X!z zBrVfIR_J|6B7_D!kHM02ZL7$XH*fdA!3bVktaZq$I$UxZTZVpydn$SLF14%CJ_nZ* zo&G!!lG7N&((_UMf@quKrWF7_^SD{b!S^(9{R=em1hOn|I{U%Bkx$bw9j-ZJw&F8p z)c19f-+e?e7j+{$*1)e;k0;rIl*kc zSJ%`HZgZ*F@~+osFlft9Joq4wtD&BvT$AAF-|1ESw|M^&d#mE=F>_fzU+mD1bW8?#Z@Thbx)nbyBO>yVChBu4R zmKGloa)cM~{hP8n{io1H)xwWhX% zNs7r|g!6)W(vJITLa;xu?DKPj+k@5F%MX3wZ^9y+I)6dr;nKyr&lp#xhHnz@-vRgj z{qO3vtb8X`0u}z*&Sa*GEJF~;KrqXNCT#@X5npRg7lOSZ)c^2n0joM_59;$@DDIfDgV;Xo56Dn zr~^IH{38gbha-f5YijUk4ZJ3;&6O6{6-9y!pJo~?X*;i1*X5pYkTr;tCaBU%y|55 zs`AdEy~*+GDry}#mI&!Qb}Ovm3Au>c!b8ljEVJNg;$4(}(6ac`=%8YXM}XiMv0*l_ zBFR*YHoR`H)RDh2JD$^YeIMQdMBSZ?RciRjf!;u>%4|NY-P?NsoHMbgX;yAsxdajF zro|!fH_=A)3NF=Tkj|~gkQ%p0?!+Bfa}eS6*o4JRLSaG z_xCiP2~(=)np{QG%eXP1Z8Xg=x?TsJoIHal8s@D6b$2<5UKo3)kwHr497h2K)!`yr zF8>5r^qb$0z%2UuHb$USCNu_x05zBSH5F{~b39}65bHF{Cch=%X_gz^)tSXFNSo>! z&HKqTvfiq%Jo0xNjQ9VUj3{zh^0FNwvAL9GR(?UwJF8I~dbBG;27}?2x)O00V%AZd zQng#{bTvQhWGTCijO^)ofMfH+0FB_7{YzhGBBSD7lWDyK`wMbGi><3sDN_&h5G7w-1b;_b1n2yqkTo|cn`+ChmW+P z3C^|MEjh$#s$I02U1iu{X`Y3x#QLq7TAOfeey?=#iMf^4*W2vze!8Oqf=_Kad42$b z_J5F6WGhtJ3gTlQG{b*P&JHCt5a@jJ8!Kr5~^W?<7#G3nz;PeSLWgWL1s@|pJYPu4etzS85^ zaX3*6X>1&7se`|{qb>Z4T=uCPf-yDT2Kg>}uvb@g5*y z@Tnf(CO-Gr$*U7$RZZu(??wVBf?=Na&a9f&bQKwyxYd{F-4<+ku&nlv*hb#-ICqsv zKvyQ(F;st%zw)oADt##-kPtk9|Ek4Jwu2#m0fsg(3^P%EA;&S?F*5|)&M|E5n3DUC z_29}g{_KxvuruRi+*So21F_SUVjRsD2`@Lw*X6R%XxAL)8A!PYXZ_}B`0v+*O%77X zG&Wgczi|A0^UgiDJeet8`sKdK1EOs5V1HlP&->0!$pBGbKdeg z|BDuw&xmtYwpfCU8IFfx(;i*B6woJS+UH1gGBmy7#jgSQbeg9_X3Aa*)8dCQSxxDG z6&hrQ`B|n3~X+CvTI8jg_r-CZn9rsSdhIf@=A=AYX@xULmkyh z&F9$UpC0Q4TrCcc_2ALX=z#} zcTz9IJjTr^%<2ABZN7(%tZeZ;pp zqYqzIJaT|16UhHg^D>R6lH~WoP4`6=UIB8Be#AZ06^1Yk3JT!k0%PkEmb?*c2%^jh z(;>O_ptZC8hsR+igCyr`k5{?N!ZMgR`P=B2TVq)Y;wOxKN;C@AMNj7R?Adr*31@O* z1|k8h0bxBe*3#1+SZ@7?3?CGJUEywf-Z@6_WnV4e8*{e9GkXIO4MTpuu-`r^YT2E= z&z7mQYBJuEKF%h`&b~ zK*BjRh6x=iaRD&)jTb5zV4&sk&OEcnxjwGgj(x$@H-9B&Z*>Q27;6MZ9*-FTJ z+;Z{RQP&r>!Wg~9x9#Cu1k~W}?}g&T6UE;}%ydIB2iEg|{FaE|7`zWlBf0w~GIYtU zh?Yzy71ifbKk65WqKO=?O;bEaSTAjfm)NSM!K#d4u8_EiWAB*V$^+=H`t{>9j*CTK zFhGN3$Mu|Y-H4H5=(qyqvc+g64~>p#kfvpB{v%ka>JQkz>{DYODMCb&9@3l(WHU_GB*U{z8PPMUh{8Q#F0!2Qy+}S zz!d zbDRmFBF1ycd&zD~Sy5J(Krh!t)8m~V?o9@Gicl8=6uq-R*iVk*XV8$jSGUjcij%|h za_YmQJ)O#LGQ84$^i#4I|0;dEUs-<$n{2p&>mH$E@lz{5m4Za38&a#5fzA$7R+Ls4 zCJ;99Tf(| zLGA%fV`Mkf``mEfJ&}Yqze|-z>cUso6HWTnoQOoaN)3T<$PkzK_7y!Jt3Z$bs7eBq z3XVfYmr&xu$r6E@vNjLE80=CBA1dQ-pH3s1MowexbgZ(QCO4qiFlnA#1ui+gOh^@U z4aQ--J(T%euA#HDZe|w)jZk}404fa%*Pt(fBY4>^)lG6G)Q>X27zJg()y$L3U>cam zNMY4pW$#n$;Bjhfr*lGc6;|z{4}RGyZ)8%?k>c2PjkoEx1qQN}L25h3$xU`G2EKDC zc$kYigxvh0y`zPiZ#Mc~DYrrH)nj&T)RaH7*948rGmkti!qr%;#^4a+uP6a?b3V^D zolqJV4Yrj{a)N5U&EHF?`i1vn>HpfucjVCUbtB_=lHizYF$B`TlSVN6$1gt@8C95K zR|&Hl6#ZJ>P=Sg#C?Y%iz)=#QFneEJLiTVcb9-z(%Gakkeknv0FvA72l(%@`LGW~b z1SFVPJH(Os1x{Mt{k{;>;+e4cq*|CGa|h4rlWXs=cRb7QJ1o_+uq{?S1d@Ze6n7oZ zu<>9HT!1wBasKhyufWm=`FZ3!?=@^ z#LJ~mnZvBN4oBJ_W1!C=F@i9g)axE=1uDUNrMo$EP~ubxs$5V@;59^v+qznhIy5@% z>?dB1NH@24tQ_8}2FnqcPF?C57L$RTEoPWoN2DY7#9PqX1(KDT)k-HZdh1I#+zK`S111e*dI8oxFT} zh@0rDw<^Joj%5DwDrzyg+&M@iLlr`4+gOp|Qy@Lp<`u1YMkL`MRIceI-s(-N^8G z!h^BAvriNnqa;1fmM0W@{*6AG ztbG|bx+wYOl9T)?F=wG~5R3+0NN)w*Z>=o>K=bHrUBR}p8ufe+tqahyYT)9}gm%j= z6`g#?K>k)~1XUkdCg@(xQxZoZRyd^botcIRbFXSfM5MDsj0KXN8|iW7!^vZoTmxcU zG!+R-RuqD3ojn-KLk!~ce~=EAQH|+jR;u7bAa7sgfw0%eWjxFE%nCvG=+5F5W7k~= zd}XCor~awe9tnhjHgA{8^U@XjZ!W?pNloSYE+k~m@Fhkk zATa)WNfQBC{Buf9)K;35=>x)E;5`edcv6uKhB#F)ORD-(+%b#$k;<=KJf6(|2!q12 z+>p}_T~gA267%d8Gp`imB%Nk}VNyrpr6S@!ppvj3KxL@aqqbhB}-uylxLz~;V&~ftJls%*NfsP~Em zI)%c;9357Qgb%kr^OMjOoHgZEGH9Zz3Q`hL2hhHN6~o{t#QzxDXWT#nqO4M^us21= z5hbB`bPY>MT-qS}o;T}^B=sO#1pKFBUcxZl7D*+k$g_kHl6pCG;pny`7%Juzpn;T) zH!Ia{^I~5p+>ys4-GS9ZS_Q zI6vkz>6YSqas})o2kjRTECT2kK^YD#uhCe8Y$;k(oBlRb2ojyEIC&Ngvnp-R6F%0h zA=^UL0l#H;)1POImLS;?xGe^k`tb)wD)h7Yzli&c>>7+GUp;RiX{<2YhlN^|0K;$z z=3fU}m>zoK4v#}n8$jG@cFhQ0sMTVU>YbP58AgO2!vU+}_;9O$ymWKuW@K&tB1AR3 z#d7tZI`@Wzc5*C&R_ErebLOeycFd@;eUxgM`#)A#0)g6tK%$jQ7jwK!+3-5_DL0DI zvP|@0?J<+;8kt4*#X>eipQ_hUC$*&IvmyZxk8dw{vy5~j1 zA5+%4YAp#f0>u4|v&0Gj?l&2R^t2>~TdmVA8`<~8X)y3LGB~C~mm%)FTLB2PCx{GD zklN8!z$8(${xyl=8wo0ijUCMB<8<+??=I&9j)Mbp!3hqx#9cM|#QIXPMWV_P>P2ZA zFU*gCok&gsO4Dl(cYC4pr`#eUk*kt2sqM)x@Q7oqDRWCrRKQ22>1}fC1irjKJr}uj zHSOlbWMZ0Mx*A!~NZxDYn72=r(W!m3Yst{pZS&5z^`FYcf;G}GxgdH{b@Kb)<9-mJ zLev!P&vKRI;-euQwzeasIx9JK$49YD$Bg+)5$HZQ_c}1EWkM_BB%k%WW>M@>(8IgK z^UHSqdg%Uvydq#n7ueeYH)%GP1AOZRV~^M7=>KjQw>- zw>ZbE0Qr;bSJoby6f( zxi=w|78dfqZ4g(S%GfE*y$^<90VFKvW@V_=c2_KaE5mCo*2KleU|UCMU3Il8hx?q>ZeZI z5^03SFZ1p)f%h=4&M*9#7grI(l3L!EggiG0Jxw!^76U`V(e3K!ztfIjAn2nzqz~Ge z5s4?upevmGX}{=*GUy9WYT}@(1wjg9f66FoKpa74On1*^h*UKBw=##9~ z;orP9qF>$C$Na-NNA{y#8Qp60W#ztAE!d}fz3SqS{t&in)sQbLMu zlkSl9)jNIVEeCY~o2CzQJJa#8XZ4)SQA#43v=lUoDh#C}&_q_2LSoxhAYXOL>)ID+ zQJ*MN`Sy)8H*%ghV%S9E?d-fm$6=q@{>#(Kx1TPFp2)82+0hdkKS`n4ucwFXvjJZ# zfZfN+whrf@-l6cj(p`oZrgA_ild1=V_vHHAa>;^CVfrBU)!f-NxsDpMk(3N`lJ-T6HMA+6>)GZ}MM z^|HVr_Y(wymf~cXDS2=#2h*VYko#7uklpwFWiBgJ19A;op=ZG&r)BXxOVzm3%3o8} zYiJ+k?;3Hmo4>=A{HR8z8rbo>cSzx z<()Vf4q{tPZ1ihvRRvjQ1n%Y%+#PW&x{MVfP3;c*Z5t13+*rM45wl&_%;dA>VTqFs zTvmK%5cxqvrbeTG#sS4}L7g8PP@8*P6<6fBgY!1ycXDaqB#9SB0u%j0Ic11R(vSns z1p_%P@pnz5sBkvlhH7g%%3E2t0q<;`)K#%RMWYr#k(rMuj|Z!C@o{qK;Go6haUdgWy%Z%Aj8Ks*shVE~Va)lacml{-J}8@G^G-yvGq5 zG@xijCxxH#Oh$M)6o~@rKdp(*aN?YD#GSBidJCS9tw2!MoZ7jkL?`UykhB$y1HovS zT^u0N-IsXE=V)ykX@dwUMt-IYzcK<(J5a-t?C%P2_Bzm0WUI;n-Hof$iB_>LwG2Ee z=g_dI)M10fIgMir=$lZ5i8W`mL;k=%Y0+-rDs@nP+Y+bQuYTLscwhdM{Q7qvL^;#{_)`eO=U8{v)FOJR9x{IWUPu>(+duie+xJ%+tm90{=JE8e=?lvN zBZ^)e)9TLZmHJpOg~<}dSIg6ic(>IvcdZv8v|?jRqfXgT2d?n)mIG&0$8)uMvi~KV z8y%&Ry*39B5W!^}13b%V4gQ<|u0>4D-}ChU^4#ScCq4%@mCtA^J@*wOG(R~`H-)Bu zA-8Q!fs8kFI#kNik`(=5L=b|ZHr^MD@NH7RHHI{rwGhYW- zEoE-eeG;Ps(l6UK0LK?JM&_5xBma%Uu!g%o@+ijZ3Q^)WkvAKKvW9UlR^*^dqbeZw z2H4~!mxx@X0TRO4ll9<`>Z{ZNk%JSzC# zra!xXl&_`R?*!nBZ;N|ArRa#+;=ZCiX9y! zD&HlKy;%A|M4f66{Febw_WxA^T5QiR1gB9>`bCE=m$*Ox3Rc^9-CVyT%#4aNWW6X$ z9p0>I8&lc7R6sHcOa2#Hz~E=9%EZ{Eym%TjPPr!+`RwtQbT4j^W+tZ-|ASFvqy~Ot z(qgT@Rgx1{;{)Pcas4izZWb{Om;_lyCe`R5-ZCu5uWxGek~+=~fYV}xGNpA(P7lN5 zg=*Eb3>7dlH1QcN4dR4t>QHsQrjGMwb8AbD;#L=voc>JNbOtmjMOi;#uX&oOS#UGK zj+K~w%#3Fmb%Sl#bq&m3xMNXS>Y=v2f36XzordGIcddQpAwqoXl%}YO-xVqlV{5$= zqnZdZZ=N4=tV)b;1nKoIEWRz@!95i-8_V^#4?jOpH0i*p#IS+H8*gO%K{6IZuEZ-vVLo6k8aJ3RgRnpD@r z3Q@mo`6S?Ut;`AEOM-WuIZ0()%&CE6z#Nw#$;ES-gP|h!4ElXxnkEHv9wLLFgNn;0 zESfnt;zd@_`80&zHb)N`fY{=2#eT|FreH+~C5JCX4aesT3EiMzp-9>K(I)_9I0qo{ z{DWv1mD^>^7fSP)ccTPChoYUpTkPqLY41wVz`R&Y#*Pba8sOQl2Dw!>;KWMx=P6PIATm6w|&(P@dMyGNzl+k>Wq>J%2bnZd=Xm z2J22Wu&}43Nwp`Kf_SbG?&Mlen}C1*R#v(v=J$jd7o5(6dq{;c9qUonmZouSo#MQ^5hJV{BlFi;LLdbT9fA%Hspl(y6xWGoO*N3K#HLftm6;2 z$?Dr=fUmB4aB$5K?|&QEm4%vPiX0OUldQj7FsPE3F2&I-bT8L7(ij6vM%1xAlzaYn|yAB(j#=a*j?Un@Z zzzVyEL2p`<#v3{&6#Mg__`CPiBe6P<=J3$#&9*zP`K|;UJ2u%RDg{@UP zyRuB52qnDx4wh{uP`seevO}x+BZ7Ij&`j&RxOCJj^rtV4;uhf~>!pc{aigtDq*g|1 zO-m)9D=$~Gw0w;e^;fHbXwfUX^6@d|wIc#FeW7P>s`I*%b1gjbU%EmfkOlf`13J1Z`y8!3T$k zlZt(i`2(^~vFmM&X_FTr}Xx2=X9i zxwrNRCSe!$W;k^M&7oGx_E16`1e0p!dY)dYGU;IM1xK$m5fY(v;5Gyqb|3G|qoTyrLYE-NpLg1T^`NYZ2%3L9Jadh?bpGli>ZMB9k z{{1?K@}7F*^I+YB^FL|h?t8Zz(i+pRR&x6(3J8%EVk;jfM-&H4^6L<+WoIS#S?U zM9qW6GRW8$b|f^Db3JPe4Kab;R`vv4dv3Y`vCi+OiKmtg1bNTYduI;7dTg|y_j271 z*Dsv`w-Xr(h0c!_dLAUe!^oT!O_!0p+qR$lLEwWugXjPY>;MA$xJIVx^(*n4Mx=%^ z)zn`Z=3?(=%yog?)s{;?xWc^R=E9K+{&ze^@JwCR zMf`!MlYi!GORj@hg~9AMVaFuXI&+Rv)WNA|H;m1fp{3`}s1t3X0y6e#%v9a0_dTA? z)swc6!qvK?es)76+*2C1c@>duE^j%1F*M7xf*Y<#6{6pRv#b3lz*?0p_Uh@qD2;YL zNggq=XCqhU<@d5}ZBnZkdj=Q9pW?VCA`8LXspCvf*+^erq3T=kcE~X`X1@ev3r^=> zY&8&?OL9_BJfE}~?^@gee%ieR+kLwe2qn;iOW zW3S>DDK1spGRyloo z*uB$3^2m`ti?AD+hymJ$o1Z(_r3Ls%um9c)9&sWgHxM?7D%&3E(f0JoUN3|5sUsKO z{(05bMwPEOKaLISyt_z^RA24)3+AgKPi4YD z`2qM3Q=9Xp6^yfu$|6qAlU`@&${R(3aBnlnd|O{lVB#sblgzOMR67USJHoQTzqplZ zt>SaoQm7&=L6c6A-Z5n;GoDrGWU$V7t}-xuGEoor2p1sur4l)ki=mUl@1|tpnso#U z8${5iEWj7!dC_cRl5SluOO;+KA+Nw;a-oSyLfRlMwHb%j!`Get_l-sA~Cml*^~8~bpQ9>N9E0VbF)n^C+FAbFoB z*QvgW|+3j6eOSl4tpYRS-n-pe(#`_+_WxFVE)8J3>i@9lJ3N z^Bmbd#4g+SS_a3|DM63hGIP#=#aahq*^&q2H@~YTD}1R~?PmyICe>N7YWbbdhkeV{ zgic4!a7Fih{hWw?P13d9slhLqbR^cLNpyz4`{CLCqEKY9u59_)>aAyh{nljF@vrHJ zP3K@Wtih6eZM#dg>R|U|tyRyZKsPkDy0iz88_?Xo2pa!k#44}-5kqY_`4CsnALdTn zuTiXQ*wRs)NVXE4LyYt~<15~d#wDXs+9VsH*lpaSnwe?0;>zrg<+%{m)bK&P5WW50c{Fug(GUXwDwg^}i zBV&M)xj(z`YMnD6)7z&*;Xe=PNnh?RS8&-rbh(yr^ zxkYmP>LFZk0YD!7Mq$tKyXoHGqorC69i*Zjxl_JU#zNzmY0tr2umbBsuX%iieVrHS zn3RjmsPD@y`fEEP79=oZj{_p3hKaKh5@=}9NT3;DZaj|Cklc)seVxn9BAcD_jc;9b zsG}8jqOArih7O?K@rJ3j51rzGzXq|XIJbs#OvVJ}09$WiU%hM7;1NcAY7)glsea^g zX04h+Xel@jc1(sS&26wKV1y2^p0OyCwwa~b+{b7=^|(nRa~SpuFEc8|Ly#v>R~G(um>U+h{sG__ z(o8gL`=%dgUb8%3?6zpbrUGbnsHuN*U_BweIGX~I`}c*D#)Y!ZvRi2@GjZK`fO5;j z@B%U_?9;^$S|1QgoSUi<+!?t&OS4MmWI4oZz(Ppi>6^Q6*K89K(P4I6oAC=^_uUtX z1MAe#qTYTP-1bNXnUQ5~4{;_Y3D8ZnN%Wy@Rf>}2cePphN^voA1E1eljS@?CnJOM4 z99I~cQ+bXbxXV6JIkWt41DlY>;RW+$8K7~PgT@ok*kzVXWL;@-M(xd2KO2OZ7v$b^ zSF(`IunI=RV?mH;Cb^%J(tJaf`5_#=3dgmrB+nI}4*F(#=lYuqL;ffZ*Ewrb|5B_k z2?>j2ZmuY%(44!Od&Q2*v{8Fed=k!z5AY`6?`1aYJ^7pzC`WR5M#r0ym{TeVhaa3l zlrhWv&pG70LAMmZHJBkTwGmRZc4)S-#R{~u;T5{iKup^y3>d8lCC2Cld#eenjc>N5 zJ+pq^4e^e^+@zfr?$E&L`kVd=zm44Sx$_B?%|bqT#EMe@`1)CI!^s1lUh-AjBm2Er z^?j(jtog_;zo@ddX5<^FYNuEJF1p8Heu<6N+S&*mQ{rfqMZu%_o@JTIh&(-*%SSkTltdD5y)5%$Umq61-+0|3%N3Sbec)ddoJHw z2t@jDc?ccS`A&a1@3LR%mJUint2og`5NttkXB7!C;$N$xw5$-XgNfOTKo5euH>!ux zQ1jA1m8A+vHo}io;*aqgqaQ?*R1|v(0T_|?e!ZLAeXIew@BaS6HnbA$w&D!Xv&cld z9qd$fco@C%qch36;kWmc0Snz9^51Rb6=AXgxK1@hY>6uoGQPuP*f!{t2M?XGga6|C zRMX*ClO3Q@*A=-rfzWFG4YQlHPE!;G9P#qQES)*nh%R&5by}23ec`E|zYLaj2tgUq ziarpF8`2HU6gew%0h0u#=erNy>H`qzftMjM6Df0jgE{>4=~lGF4`ka^tv(U;W#0tH zqvPV8hD4)VxT_zyrurUR{_vPcal_~F5xFmawG0+TDe}M$-i&Kk$x9uxQfoj_2)(y0 ziu+@YZjQK%kiO64GjU4*lhpNZZiz_iWE7mc>^V~{rNgag7K~r+aw?18yTxS>wA$rT z%|$0OY^bvBqOVpa@9@*^5L&iV1&8FOQl;P^*JholV*`d0HpBt!5xy00afI}nH zua?SpQ`TB`ZBBGc+i=GdYuOi#h0?I&n2${n7iRg}Sem-F#+^SBNjpWwa;D^^Jo(4( z?dTcbaoeKh%SW7f1XD}Zw%`B~ynxPb_QWD0jBLYY{iL+!dd2d;CyF}%|3~x6KX)5XTD*CJe!3G4MjV1IdpW4XWi#AE|`mtgdd4fOPg+$ zEh2-h2o*q?ZpHPUHW#Rw{|z5=qd508O&w7-ejcC3v<>ONfZ`9Y#1!0sApi`;SpY3S z(!USzW<5-(!=-`X$bna_Xs?RP zGMqdnD9pzXSoNY>73*Kexg1HE#}XNv-6ik900Ceu<&E3c!ynln8*tHig20bF>&{w|NNttUecdZ#X7|7dOzOz<%?oi$ty^l5$2s zO$F@rlSpqp`U{Wy<@gN30!LtmhUK!Rg=1oUV8q*6bSUWzysaydPs~w&Mdj4YQ!T>7 zcRyg$Xvs4PB9E5Z?ml(=MD#!{)Y_|xT6Yk+ zc4pTgs*QX}5!QkQ$})>JrmD0?rNZ!Osl4RnSjSxm#|6PJ3$#O_Gwn%&GnKHvMF$-@ z;V)auGIcL(I;SGGc=pF?O6+@NgHIfKxiukR%Ye6Run*p=3)bIJcXyABU?Lmi_*myZD7z$b=MjstLC9 z3`>8q|5(l2lbjfPI^e}-SMbSigV&xs12bHpY>rg}JuYt5{~nP;!5RSgz1w=gbzY#Y zuuyq={r-}L8=IYSjs$XrTp&|N0jJ58T<4Jg1nEH0Q(m{^$+Z}%Yn*6Joj;~RtA?|A zF|Ts_XMH@yxs_~shwztE6leedE&QB&bP&qgIXrv~Kgd_xsAYQW=Y{qHe;jTG1JZN~ zu@x1;td*AE)!xvzcr-4a4a!U0@eF36^FwzaydI(vN?LEqXB6Y(K|ydQ5Mg18=#fju zE{qm3b!?w^qSa|Q@O6?Ay=-NS4=MYR+;s=RHf4N)MEdp^_(aWCzP)YKb00UO$k>~~ z_qLAT`9qI_=rfBPo&On2(iOeW#=_vKDEZLR92{X1`fZ7X6uUfq=`%nMmwhNcWmZi3 zc(fcOp-vRo`k|{%{TM{JN1=`h#_@r|u$|!!L@E|~^^*H(Si|yot;%l#m~RjuxTN(pVfjv)sSP{e4!3BpjJltg_e>#ovH(V`}V&l5RwV51%-NM=yJgirli zd3{2_A^DjNvC5+nwKFN5p{}2r3|1QqgShbm2{OJewsA0AO@_-j6{>08wd;GVS^EZ< zXND=_ztpu}M+FqHK)jMOyR2^vPeyMmRT&Cqk48JB-BbFawfioP{j>!#;MSJdBUbbnk?G8N(9E5 z)^mVZ>vs#3_U;^iux}*r7Up~9*pZa(8-FEkRg>9p!eBD)3y$|gwxyiM9x2?GgNxQy ze}UYqsKHVMF+9L&r8r{QL>Os5SGZ|3e1jrq3620*0e(y7K7LYY(Wp8&ev6hj8YP9r zhjmNz8OEbPCAcUjGNm8@01`j#6Rg>#ON=w_dyAYFJcSv1CH}gzvP&O0mAjClwFgHs zQ51>ROF(uYJM7BYshLv*dU;GB%+8oF5Z#MMA}3%m7d(byrU^4;V%DT*U#@nD!h;!W z>*$*=b$w#mikSsn6*lsXu2sl#lEJPiO!`;_&OhZ3i`GTru3)Ydt_B$szkd5CL_!k%Ld*-gfpjzF7~OW2*b7Brt~{9YjHce!HM4n>ta zCp90Och?Y3^!tIJ9e?p?B0q2E-}JlJ!as+^qR=$AtGYrV{t*HdI+c+&tYyE*H4Q&i z+Q-u&V8=mUzES*+z@LOj!KT$8uzd|*FILp zC0t;Xe^Fpybts=P@Ss=_Y^5K;!e5+Yh^PN0|1h|4c0t856WM$(i38$)h@YO&KaIXx zXm$D{=2>j?8OU`w1|p#P#{7-d>kHWr;xJ9Df)f$0a6~ ztar)1*q-ZAk0j#s$maqlOUdDT-jpZ?3Kvq`TVJxP2MqxIx~KRQUtPV>tR3?H`)t?ROQNNNk#ZHoLE#m?9B( zfe>86^YlP8A@#S!Qjy<~;Gu-u?lJRb!?luAYlnX!J-}C^&9r9#8I$5uyRAi<2h32` zVKqd3wJSbIu9J|e1xt66Kk@kEbFIeycVX|d#$nQpP=TT(Wq)FM&>)s7h5C<@r)%Z& zS^)AvgZH%LJ;b9rffqBpEPyraM_t{XUWUE2ox01h-C{%F41Ha_s1z?CHpNxDkU2$`A~iX(bCpHuhZShdA13PXK;?`o^lXJ)q*KXc zK}*3&_S743Jud&ogt`^pBc>~41APe56RN8*C1Y%pUJ##?g?7BOMshpZ%{&3%Ksrs3 zc&((@h00002%t`H`xO4PQqheGYr%bYIXzufgVBf9czQ+WRp5gv& z1S}^K;S`TY6+383K0RgOKvj=n80M#`0HgJ1nt-->$CcuRjv-h_N(9kJfT+Wo_!_}X z4x>pKB{5qP`tf_(%kmG!e<lmbA1`5hMfyGC5}QY!px`P%rW68!P-AWr->XjB}YANu1l5LYHc z<{8uKoJFcT0Y!`SL4LUeC#9j$Q@Q~5#^mahI}g*YNe>gBtoummh!muY{m~UJjCG%# zDb7ejyXi#u1;isaq=dfAm$u548Qu)ia-h(mXaqcRy3`cdfkSqa|1s|iJMDT(>7H!6n&H6%Aq+s9kWj&T&utV$|wj9DH=@ou0m&QDkQ zaZMId>cd!X;ET|-%H#vVB}{YAuGcBV&ZQ-8F==MYof)L|rE*TQ1!N+sDJ|>$)yKlP z{o|N>vHb6-RPaIW&k;bqQ!$+gPkipEm7x~Qk7kI9+8f}6 zxm0HftB9|(3j2H}Cgj7f=TJPPI^xXBY9W0u@KJh_fD%QWsIQWVzX#&hBJSxa>;M25 zdPL(?upp;}^e~$h@t<$u?@3b`w68xZ-Ws4V?#ay(VOT^Ppf?p*8g=761s-HkP&)7H#;PojJgF2P8Si*oHJM%(Ml)V4n14g4A>- z^M=iOV61Zy?)u(uPuceOiZ+$yUef9QRPwb^v}cV7_yhjOf@!vat~*k$3>A+=O({># zXQxe|2qmBHRM=?Cj)lMEsB}S7tlL(~A_h=1Yi^{odcHf1SC%orSUxWpIIhS*fQ|Hh zje6Hqd@|rGazF00p90sQZfQ)*l8R8p?%|J@8dn_e81V1R#NQzDFK8d)#zV3TZLzDc z)_WrIYv; zswyIMl9jrL@yIY0PWV6!!;%_Fa?NDP15yJhHFOEc001+ov@wxr24yf5lS4k$Qa4n| zdau>mAsZUx90C7CL69PPWxK+UgZZ4^ zjl#&xkdD2@hymm;()B;V7h&(qJ!M-&#|bSjL(9ueBfN3D&01?D`(oPh zZv2`(o79lde9(OCUIWl|3`Zny@fV&aSWs5%hUaYfKzUE8uy6pD3e2YhnS2i;*>pu7 zB^qCp+TB`=TUln}>yjPDUjP&e5kmXBzluJ^VID?z$}sXUK(66+o!bLtR1ly8^&ZJd z>Wh=hE}yFpu`%g*gl_B?*>|Q1rv5G2t7JWca~`_qKqFPV>zUl|JtZ-NjE_jR?7OA% z+MHnp_E^)aln6}tjkB-DY$>5(F)>}m1k|Sg*!$~L)O`7F%~8_@6hU1(57eAEaf`z= z{V=QbO=_H>HI^&1$`w>yW4n(=J>CXK77|A=!t12NZKOT@0a#Z(0dOQ(WR8PLkjRa7N><5qk+#Ghz}`0A0n zO{fGp$TO_H3Oo@M#44+gLk4^%n+b3?!fMzup-5C7!X$-C@H11AKyUaT*j}%Tujhz|5IIAm-HTCnRTv0J%$YO@pk_IG*tRZ=_tqV%5k?P z2eYWKRL+Yss%bR1-0w1x?r!$^uYuIJt1H%PyQlNvHKpy}eH`=rayjDq2vy`RKZ$>C zJbO%Rc&R+xN6eC)SQKr)nra(sDVYlU{iTAl5s( zFE<3uI;FhTXipRWT#+%SXMZ6gdp&E_IX60A?fKVW8FsK6g#k?ObUNE&9wW&vieWTL zv+*BG#uqjciPu{BiIENSfS%pt*m`S!P-4RDSBoegM%I|llFa$1zdB=cf36KB^e>gcmX(11M`qeap%?x6h$RCMP*1m0gq?pU{(O2Ex1}fIP zpUKFTb4r`8R|bR zUmJR_2gj@2;j{6P6iAFwy71{X zn_uIv_nFuCyBWDntw<2H^~dbf_fEDkuKD6CSAfi8*5Qr{-t(T(i|Jpu2*W_R+`~6eGf~;)vy*1s)M@sjGiw_xBUi8DiCDHZuN9$40&s0v(6yHLR z7K$)26dB_XPMn5O>|bg0L3w#%Xgewsx(D28`5oLL_o$+?5vfKP02r!E9)-z_W}hN1 zDI+ye;4x~`;a6_bnEfl4NBwA2ZmYkh=-Xt?+@lkl9))ViEzRDEmxG2lvKyJ+<|==Y znp#>5g!8m&%c{Y5V!Cmx+TgBXFze1ZXj4X3Hg25Aa_?xlY+m6sWb@&4e^WS?V~`+& z_LBLUmTBKPA0<-tG1^Zu(I!6mZ%)62C8-6UrG;HSYo`E&DE!F(SBZ>dsu6%pKOt17 zUYCWpqs;5ccv(MNJ`=3N40fzG$kQO^iWjAlD?X+Mm2LmK`>(D(AP}NQ!<&;Ta-$yT z-+@`Cg~QU>y~xyWj(t>wJ)j9_TDhcL2URBW$rW+WYCIs+JZk8Q4Y-@ZXwh&g(_YG& zuzyDvsiK<0GS&gL$Xz7`Hq!AB!FmA+fFch>d~i@y6UKF)&!9R#ylhDDI?6KX8#;mM z92+)v??KE0N!1u>f4N5l%xvY4)u&x);_)6~pPLKwb~rh>J<@IqUA-d5d?s`Rc)HUM zj^jM$sI8`gi3o*5+0Ss3NXkX!YySPnmKG<1tVV-ss%qcsd5A$;sol_{Jij&nMN_;J zeBHEXnWK)lITa}k*>LS6H9*V^yjm5(onhO+75L{5gbz6Of)v}M=O>s@U^Eq&p4KtV zrzpO~__^j`Hqoj23=mm&JWHpAmHR1Rzd(UKi%~#w-sS0XC2Q=8ql&ivtO_R3x&iXz zNJydlU39jJ&9-BZbknaqb*N)c|>D{HVnY4v|Pl7?zxV#HQfUQ-aI zPar{}&Dt*-?v$PN(QB12i2~#l)x=!qNn|RYG^e}x1Mww%SZqP z$>s*MWA+msJ~zr33d`NYl;lpMs>RIae{@dXr@gBZd_O9BH;+#5oNLaf+73BA%yHM^ z!O~o9b+lIWl9DqBk}5WB104WC{bg>5T@dA(dt1hLzMJgTYRFK9&$k6^3l&?V?M9Kz zl@4OR(OOnr5N%q@o6!DI;w>&~bb3VA6BHdXhAX-b|N16!*w>E+8PQ=%xnC&G^?1^fj4Sf!6Ul$J}sqTNdHVXDgX-^ccu5 zfI%J%EEQ-RX1gDz;J27XJ2NQ@(HB{5|80#l%u>c2p1VW)JFOS&`J99@3Jm*qCNs*o znF6*i-Es6h+KhiAfMhDb$)%3;i)M-t4D}}|>)9>#4vZLIhOSfq#t2Oxn&As1U+-3l z;vl=Q>S70{DOg3g2t+^pTY_FPXHJgWGWe_p)Ua9O?C3>8o1&e*VO`E4-Q&NPd0r1r zj0(Qf?C-E*a1U2`B&hmw6^KIQJ`~?R09mBibJEUe69ChR`gJAO=3&$D2>6n<41bC@ zC1_s^M`703eB8I2GEkYjJ2VNb4Vs>kfWo4s{Z`CzD8bUhXkt#h9bHj#J}~TDb|V85 z^479K{KKBTjnl~5N{$=)fruQkc9)vaxoB6f)+X-&x|~Ar$X)#R06g$?s%&t7duUJ# z@L6`{6T^$erW{pLuu%03`74J-vTCa#q-3=kWd46(rO!?)r2s6xlEA`@hr;r!;L$wl zc?jd0jHbc{ZGOH;6=Z&C0@H>SyMM;`ZsUuJRDuxj*}>Cp5QiU;sD8$XIac)m^&G&C zaILmbB2>t>h!!XY(9AWZW?7e5!GW_Bd}|4A33pd_kuA(xD1-=M>V_S?xxh(E=6d+1 z?%GqH7#w17Z7)5Hf(z4jhRFFD(#o~k1@m+C+EIJixfOxme1?Kg-d`p3hrA@`A|7rC zw^+Q4ejyHLlEQPM(UQ>DHK4Y*4pFtSoaut@oF5pKb{Pbhu4LVFNp=QPg-Kdb$0@^; z#-ItHK38*}e^Fd&Lc!ZBOG>!D!s?@1nWlM1FH1{DwAMs@OU5ZHINpHXNYVK>E(C{E z%t?VcG%@xw)JFIB=edmDDqfF#N56}5-Ekmr4qP6T1Q0B~xUj6tbHngixPe;D{k4A3 zBJ9=%w64_qO9TE84?xgrYIhXZhQ}2En;~K$ulgOK$U>&=mB|)v6xz0{($@+5ENkI_ z0x-2eAiOBHdG4J&_pAflB39Fch5;g689)55mX6+2=<}`{lBoXfnjv9f`KTiV)HyMB z)up3SvS~`C@{qD#(~*#W%5Ihv6iGUEzML}T617akDPaMIYOMn99qEtuMq&0(n5(R| zKK;cxI@j~hne#k1Szm_;Q}Qr8^BNj*{NkzS&<#-7j|gj!_ym5wV`x7jSXaZr`Zzll zB|WjPM5q!Rv?1pX^^omPA?nhi;oTo?>}Xd4aj`iYM@2bx$$a3O@ZYCt0jm_U8w zkjE?83BRbGA_pjn&KH!{p%3;_ODHkyGrt7ek**Z2M*zojyn~h|9C*0~t_n@Jg=o#) z^^eFA6U^P&1SCq7ncp^O>AH^HFssE(Bno)8?R! z)##AEnOC$SKVNuO6WWlTA~u@rqacfgDdY~)7P1k3D_PjFLgkqu=^{ALy;O`hW(i$U z)x_~mi0fct<@tZ{4(bhf>NE&lBykXimj8iujf6BY|ZAurX%K#_A0TR z{m+LHMfIJt*JQ2&Iu9#apNH`mkP_@%;P-^%8_>I~%@bE3H1J;`DCJG(3-)~a*q#US zp7E0*ok1KC8-D<*`K}wn-Z+{jvbV5#ke?dFO7m#|Dyp2+l)@?}G5=+?a6bPz2=UNN zKT*(DD5V4!VA!dsd_2o~hu=c_tURohT@V?bai>O>A)o6-Ry<69c&`iUnGg`{CiGaQ zgijLl;K_&50uZEmM$f)QUvjhh^;a$pm)61kGgE75ZW&7kLAf=qp%UB*zFeV%GEZA-tU*o$QFs$^|{Y`<%}t-|ER(v#PhcCYCV(4y&}21V;mpBp}tUP zu`b`p!Iy3UVZP%y8F15He7o-f)4&VB#e`&-n8dBSj?1G7d5heA!ys&EalR4;0z}X} zm0$5EpS9s^k_j)HEq$XEKf~>_Fb16lPni47Db!6G+Mn=WXMlA~1K_9IBHrABI+UE6 zRwTy;TMcQ6m>yX8sy{4Df7RHMTT>nSNz+)bgxj-?yqe@ZYpDrA= zS>Z1H4}R|gVyTc$%ZJ5Mi?}24LO5P|1zO!VYR|U%!u;E6=OlTD6>gll6ifkUz!1p0 zsflmJ)?<%Zv6j|TWYn2G*8t+9d3shQt4e#hsE20@UbDWH344#wxSC2)kvaKunmErE z+ksD{D(<8tf;a-+8xfa3Y7ez2vW8B^UNQfy%w!d zeQ-jABfDs2;=Z{uhv;>3-L&u6)`0jW3UGEL@v3o}NsAXmTRexQ4c+%=D!Asr0ZbbX zoNLP>ELEJ^(fn}>vhKiiIDI+1F9!DkxrG_Kext4y*v8afkC-M&mB(b)c#g#*q}4o) zpRs&WR#nWfTh<)%k=WKGH=>NK?!N=8S6IBhd53<1D(N$m^H7Ie$wR9;qAj3Y` z@BwW){eF!h>R3dx65!{6|1nnB-Hjc_$JcYPD`Bs(h@$d^HcVx$@{}g1m$p+a6&>|xc9c~@g z{J(I}2t%|i11kVrOwsUADxtZ=88E03cx%w5X`REr;o92HmPy4kg=T{rs#nHti%Whf zyoj>Zx2kanp*K-O+Fs6>4iDb+w7-k1h;GhONI;5_t5&aEBm<@lx-=tQ_b$G9_XpjE zhgJzr_F@q1ECf7opDCP3xfS&aYTd~}MpfZfv0ywi}G#-E0+*vtA1)(q&ASU=hIBKjV_b_xYq zdvenuc3ODiE3O)PY@u}xW{dO!$_GE|KcOg0)TG@JzUxmcyd|xKmvVIOLg9;MK!`a7 zq$rYby2RCVq(V~qv&CsuK)@v!YO;KD%7-C#-;w-hVz268Tq{`+?j4`<0= zB?SW)u{ULUt^e9FhrdFPd5yKruNCgzFLWnpE~qJO=eR)uI#2D=%jv?=d3S9L8i`n694z1g zx!kUB8Dd>q^AjjSS}LGU5a)|eo)5zny3iPpB6VL(-4>^yao8W;Yjt@7i0t~mf$|z# za?CTj^=U_T47!a>N~f|^ixnqJQJkW=V;ZO_Il6tjU4^2NLhICq zCH`XUbe|%2OB0^xmXeH2hb{1G$ucj~k02x6(&6T9}Vbv*WAPmRcLktQyl1DAY;(@NT z0o&jV$O4nMSAG`GU?)V{`uo%Fh=)IVHZ8PKmfKiDW?_6Qg8vzWE?!_U{$V}0q$*>q zv>kkZW!;UQ&2NzVR`>{Js5m)DUzxBe4$6tuYa4&3t*95c+y*Ca`Kuu24ht)v9?jr^ z@hbCVY-7&VrntR{K9;5_bH?2_aO-HMsS22gF6l)Nd%cP6pEpA+{YP315gsBswudC= zC_OQtWSE{|(3RNBRcPPPDDW9P&Y1rz_}W2M?tO=ogu!rCD(^zYWzh(YOD$}%fRZ4t zSKrHW(nDy3iHwGml4icy>@UQiILtS3jvYk6Zl#4A48N3eXUS+dgM?ay9cOp!zA2!d z&8uuqoSc1AkCD|NHhu`IKbLt6h;93Rx49(^dRxy?EMAxL3Y_)9sUdEVE}V)BsOP5` zi~mC1NL780{uh zFu8inZ4p+9RReeL>!5PVs6xtD2mM@jJiz}SP}<-BDk^swOHVs%Olr4qbYE+y`RvXr z01Dy=nDQkGwY5eOLRV3ggD2~e3^DP7=ddM_5ylpO*K?xxe-npq$6Rvp>HTFruv1`) z(GyTPZEJ$mQ(=C^_-M?fABnKNo#N@51YP@?rgkH_B_I<$RZSdA1VkG72OTNP8&ROddGi*sQhJe zqqiWq=I%%XrjzNBpP9yi%v5Y7i2Vx+19!L4#$~4@Nelr|J*zJq-$49dwdzs`;%vk9 zS{I)}WCWM@aLd=KJig}rG5DTdQ{b7tDd2h z`-QbE_jXR-39xG#Ugk52uJeU`rxMo;ei6|BWSkB~4h>WaX49!my^{(I`DFIyG!#Yr08@@;29Ip}Wzg3*6V{5)G!jk~5pH z4&&$AK~A!sC{uT6!IM%*KCtn)W3dOJoCdaV6vwXs!<%)N{~mnK(G4;ia)=vLp*0Gk zlM>|1mZ~|P&`_kiUbbRIjR49KE6bjGu&`LfJLkVpR$M?+{>wxO=soro{fY$2=M^qI zBkr`%@cFfP6>G1gr1MbKKhhk`nRV-Cr7vXk^UAsE@L5MF=S!7pRF5eJT5QgoFs`)E zEIjqOwI#81u{&uw`=7LZLtF?#bvT&NyDp*KZqcHio9VW#ua}AAk$6}2m0U}-2N!*Xje2Xel2AiQ3)$?A7 zm6;EIG7fO-{Dh0%q7ouh#;B10a1#PEA;s$P+I}5TpW)`pB-9EG)iw6p0;RcJF zIuu?I<|sSY+O$tDuPQ}M=w(c+#~Nb%|5nZ;q-etL9m!=w{sdltL+d|9U?QwD4tWg7@VIhLM>+KTuNwmYJw0|;fMiWu@iViZE zvDM5ZNI#WMXF-=vT%v{TA*5}5&u&Ix0#qcQnp>6+bU#Jv)>;2Hjr|rX5a)l47DCV4 z&>?cpj$(yt>z`Ebeoc25(VhP%goGZ~+PVQEmBe3Ei{GnI-Er#z`VTSYQqTG3@Y|{b3 z3edd=Y3W+=6ONE=N!>e694S(L7}#d%37s=fxRfue<X*}{e1@_)XrGW%dApj& zorWoWoRdq1_C2YFAYaMA4sf|SXZ>^rs7gCY#e_AJnCS(Db29_ZKJNIF zF|i69om86c!;Im>t+s54h*xn`#%i?yMD8T231ev+{ zDa|Il^TNA-)CMQ@fyJ_wYlL&fhB*$1VX-a;s&WDtgVB%Xv|=>?@JQ!cJ?2v*#=h^e zOep8Yc0o6b&Htbm?>~#RO6YHNvlebbJ^g6jfNG13WOWD+VRZ!_7#iLhqNf+20hG&LX}$4(q*kZp?_MO0Da-z-=;`G!7+hJ2<76B(!LtvU zn?LDm?IZ!0Qj(79jg)w^LC%6c4LZZ=Pbqnsa|Vt(P2FalyAI|ay0nVcY))-8aAS0WoL0o` zdPY2r)&@(c9e6gt{*1D?5CfIxY+@hB!CkKApwCfHoY0xB$A(+JvH6@*X@PWL_^E)- zpfD`f@3iix*b$@wa}eSrX*DWW$g*b=nP2NkHh4*rtyh9O@))?x0Nj&@%%{lpD~fzn zpnLo(UDKYs7E+#&>ngtJXPPlcI8wv%Yr3?ENCUOES>u(fXttx9*5OZg=HAydZ=F7R z&I^|*&{!qw0iK7?B&~$Qqq)1xfPFnOFk1yb6w=X+j*JyC{xb5PW`gsyI%g+;mcQhO zI^Xu4r6=K821nqTp*?*#Z!HzBEKaeGD$rlrB<($xahFtTdgKRHF|VJ}HN%bw+vtrd zh*?LF5iMKFyu2xo3K26jBs_nQH_7gWMJZigv0RhUqkYaR+JUaL9@9#(SM=>?$B#_6 z$ReN|xJlv*B0mxkm&Q?N=t0cad6s&H~njZERSdrtMAjK;7ZO z2uinAm2;VZBEEEi4a>+M2Yd ztqZdalihss+U#a|MqD86Z|9i^N>_7^;#9Xx0QPLsJnopnQ0sD!KAGWr&XwYe?J8&U z#F1BNeXl8pJzR+fhY0Qe;8zkMUoM53liO@f_9R>IQ*p+2d+jYwE;D;dB|f?P05NuT z9geX$my{-%9%1ks`kX#YY4Hee3kMqadK<0nC=g;1=pxTjW_1#bY`&j<{09&3&9NwJ zMOuH!NS7Jm_jV?+0xvcTIhOFAK^FyamW5@@JW08!tI7IhgCGa1@{_F# zz*L}CSJ;e%)8palYEQfpTB_{^i8@duu#+eXKmZ8uk(O2Fg6t>MmqN*KVm%XZl&YHU zTD~cYRTZ!j^GZ&mfU913#?)3+US0=*;6&>m@|bt*wp%7;UAD2#EfU`nzmi3Tka1ts z**lzWpaGz+_ce705Y+ZEzYtpC6t%G^(Zsfnpqa27t5y<6w0K&V!NNo}eER=%GyqV) zZdnnDTK+5cf+CLsr7jPuC9%R3V5*3IA6nmAQ-&JoSH&b{hD+gJy*df{-C=#BClT=D`U5F3KCQK`_N+hkHnYM3 zo(j28B-NY&=)xvbj!+tOR=){Y7I#dk?yST&YzbR{aRpR0%kVT)-Q$A8y{>B zaRb(XBC!p$n2UE;m}gD#NJUE2EQG>+NJdhI4r%yq7=X@t4hn__w3ls}6;z0J!)jSp z;6`~o-hL%Y_f$6u^j|^%RNer+#ej-c9HKjy1_Zc*KnU1FiEGNSzcUE#`-AcrC2KbP zlBaTm2SQFxcxRErY;VV2XkX(~{BWK0Ty%T(HBjDZqwtIhHe^x}jt;d_Lr-IcG0MS^+4Ef07-GyUEQDLoq(G05$eoHed*3dDLS~ogn|WY{2^)yqC{bx zv3)HbgwmVf4@8sgrOd_MT9bpXsil#6)1?g_4%T%P@u09jBkX2;6(yutoizJ&(lIsA z--(`54#ksjDCmuk6FSq9K>|;1{kdnFm}s^hcHP9TDih(b|LFK}hlq%oMlm5PN~@}a z=MM$l@Kg4?dfDXCzzzXZ$(|^x*&ix}{0e${7OKmySYt%pr6lLo2q2{hDpD6KsbLOu z-oC>b?N^;7pTAGlsJ;Im6z1G*aEF0Lep613_4fEds2nyJcf)-Lw6>VAXGTbVt?FH| z^ZNW%zDX}aD=)i*ENVSja>e=|*JZCeuu;aBeRCj`D~ zm9djsH;ORmCBTb~PNpi>_mffhYkSl7F~KNBoGcVY`bbxuyf3#6JWEdOyktGOd;f^Q z*gkH^obx`)fGp>e;=>8ouO+G|)QsD7ys4@yFclAB059P+D} zhiUD_GFXOZwx}xo8&GEjYl<@baNeR6fRgs87zi>GLX*2ODae-oeO_xvEbfSLe+ZtJ z>QPTxSp7>;COqHeAh!$apmUOwm;{?fQmFNvC*1xW`UC&6`25DP%q0AlnbWS6<+yXI zGGtlV=+;01D6IXY@Y(QR$PQE{N9@**Xq))?ny#<9X8%yx#u~7&x)zmbmJV zXQ(Y&GE9@%-ZPIvb{S|s%`c&#Xl&@&pSeh;J+NwGm`)FL7#sc^#K`*<@L7;75j${h zf|9QeXD=daY0flnH@MYqE5>~CZN>cp9KRO&IT}jrqLjGB%uDJ#1*BTKX z@f_Qh@~?X=B!w!EZdj&=WLiQ_4>WVEmNW{x?0)xLgD6)%j=!N(YtMy>sGR) zs)R7ZLUqm7nHa{UuQ1f&0J^{-88svUQPHY|=d81^f(xkJi=F~`NJIT*;O;~M;a{OE zVq@?>KmVhbwMwKQrmsECQY!ZFvNi)bcgRcTgV*~Yv!f%Y?z>p;Xb7Wn$iZx}BBQ3t zcnykqpl$U7`vz_0UnGLPOpy}gjV2FT?O`qkDAyE!O_f_iBfk^hyb=;+%8~zViy$8H zr9D6dSOYI@nqrg`ZJOw!EuaHmNn`(Ql!{~tR=+l{0%+;Y9bnAQPhQmZY$sdzd63uiR$N37_H)F0 zIP^UGM<5F0bR*0f5E#++9&qDA zQa5gdP?upoDk^*Pi#bd?a89#Irp_&MYhoY5*}W<4 zw=aM@Nv5*rR$|36L#MVzI@25qeij9PrH?>7ptd|hVmu2ppG8){gFi8}al2tBeQrf6CBT@47y!GaV5NX(MLJyc$rb~&RNS*qXS{elS zj#^CC5XBDSi`!vZB9wZ#p~YG@bq5e;2_btwP{OF*`NEN&*Jt07%Nkqv*sDK?oP0-M zMv^0PUC~Tq#HiOzk_8}*@x82FVXzL?Fz|GcqF`HLxkLXJxEIccV3lr9s0!z4KF;G2 z|M$;eD7FZH?>t1ki+8o-pDvhYEKMg9BL8qTJtleuQPSYINmFniZj>QLd_a*1rpT#t ztbxa_CF&N+u+7GP^;WsoemiNAaQxnp^z38X3#e6l#9P=r4aPYeV{t6AD$7 z@GuGK4ynr0Dk=~nfQgRXylrR4!Aj47t&#R!lS=>!v8`&r6O<6-wr_o>pe-*`w&4l1 z%1b@1=52AS!h9#wLkNZ%Y_IFE6Zs6bTNzFWbCU5iEfwFR8$z*+Vx{|xx$#+v1D#Gh z9V*{OV?_?X89;JMFjEd+(^G<)FGhbW9z!}tL^f?xr3`2HtMgM5@mZ58liNK*fdSm9 zgv<>2nmc5hl<0_LlTvTC`8bGLPV7BlC-#NFiARgnEZ@I+b6g@-b|ufB6HduW&vv~+ zJHjw20wwiqt^q$e&<16aBBSj2q}9HGCKhpASp1@SlX(TAydiWWvmqbc z6zTUzX@tX4`(?!CH}VFZci54Qj7EroZjyy+FzKCS>;XoS%v? zraNXLX7qAFQW4uKPdieI()PWPOpj6Uu)Jd;nWBiEu~qD5%_UN~f$uNtTJmcqm<#w! zN#VF46Eashz(1L#q5C?+l7SULvyzbrvJ@3JHrHdDC~6u#9rJF5$K=$Z6?Ea$MV)8< zjDjOHF2gjd`+HKbe1hw7LWImpu>?XlVEFp?T_9kN5B~$k$^K8DJ4mLF0+jJ&xX#PV zQH4W%y^ntjn;h?UvSGLmPzdA?MKlQBkhc9geHQd~^mVKcy5MkX#XPPBmY~N7rX}xCB4uprbAZ5eF zYrs=+gS5EH9#o$BmKlgzUA2d>wq`h;RV4g*?d9k);O?<6#rt$fD1gP5Pg{;tMhN*E zW!F&rZ&TeTjj@zQ+tZENByS$Rta)#c>;+2b#u1)B?dnKtJXSVps=awf>fxs;Y{4WB zcrAm!IuJjUkrg%Q9m_X#1-6pgd?ZNu_k#ntpt73C4hOb6Dn&W}`dyP|Xl&T^Lo3e70Si*TI)rh%F(VoD>?ghRaCuufT|t&129A>4-F5-(s-9T8F|es4*}ROuE`h^t#v25%S&tT4CidqM z_CEs`p*LAl^G|@w>S9-L&mD&is@^|ueRoyHWU0`!3Z ze*;xNs2d#@X}6DKbH^Y?@&CuXYdEg2s*r|m#=N1RI63oQa#6e+A!k`-Kcab-VDy+x zpc^HI$|7d&USh4t;Z$X6s@gqJnzLE^N{6h^(@f@qhge?S&Vv+-S~keX7abjhan*#A zUoKT;;nl<+woNhC60wnJdL(B)(SNFb;0#w==58;<%(O$+%||J%H*3Z0QFm|q@k}cX z?ci^Ks)k5)5Jlyh;WSBqDkop$QmTXtTI7AR#%GqT$fXGvqj&9|BWn1MDMF^2Q;o}{ zrI(y4tU0FXHje4C3XXPQetiF#sv;s^#VmAn&|_nGTI_Z4SKT~l3bdJ6sY(5g4}b^I zGcD;k8vJe_aAtqnd9NLWnn|@jY@#HJa&Bl5%bZpU&595ICHLen3H9JS_XkOX^{g%+gE+nGXA=|PgzS90Y3R0M)+FE zI|svqh$8olsRw`qDKR5g$SNRSkgT*pVm82STrPT-{Ma}&zhG?<=}BR4SoZPOI~J9s zLDlc{!xzLr2rIr$LSCCB{PNsNCfOahEA3}4AJDMTLHa3APXygaib#el@CTdTc)AyvjLimyh)<4+t>L{$uZtV+V965{tldgTtbCj z3ip$PQTm>ipiu%6b2K|njMBAN++vP^zW06y{$~(iP(6u|!gh;yidh7w&!1X)_JC<* zozALZCYuJ5=~=?I+|H;X|9{6hi+o{Nzq^@h-)RLN3cC>$+2#@5oF11>MJ*ll0p%s>&4+EfA&3(B+>ODhiUK< z>T3q|Z@z#~Qjbey9fwdtT~`={u?rJew}y_`(R6=(ST$697t}cP4@i1sp_@U*V%Qaj z5~dOzSSO#QMerNYa&aeKwopNMhtJkKvDJM^UI~Q)#byDIvyp-<;gPV2gWzOlk_%jz?qhC+P{kb&OSaYFTgKC+VZjP6)Y|008672Mce zB|eOhCcBmP;rx1as<}SZu$j?RR6tGF3h%5{Nt(CFc#{^mh?AZP+x`D7EL+}#gH8@R z{TKrPnQM{nbZmrZ*)wR_Xq%Jwp`uHgj?NEjnR#W)$LK59nR~FaS~K^Rqm*k+GEY>z zsImhU1V9qDRLtN8pIc)>4&T&1SW<g)BrUpXK=NXH8{XLb`<7iQ z%1__;tTmJ&R_Upea}ueU`ZEcg{fOhB_1Qr(`bfq8pN5si6{|0|b8M%C=OoQh_80wduH0gn`k;++FK9U14~? z(tzGcU#y<(FeE?4%nZ#6`kd-olzk}yABW+s1J1itvoL$}0t`>I!}lU)RdVV}atYEd z6yA6wzhpE@!?eh1xUf}H_J_J4kNIb~a=pI-pjBr(6!6qUfJHIN6;=Qb*!j08j{4Hs zJF@vGUE%!#Nq;5EWWYcTYv_KnMKxPsU(2d+I2nD>@Y)Vc%1Uy3m2|&_pUSU;{ywAkaD<|9@>*f#B#n}`SVfLtZ%iDOS zJxpJkQU1!PGM`aI3AmrgQ#=#_R?97O?+X)#p~K`_<9BB5XvT8l^`marmcx-g+1 zZ4SwJapBV0_jVL*>9aM2MjFBOYu#l7cDAtBwhjtOF!aTddM%om(-V!T#>}pVG$IMmC;$%1%8y}t1Ya*+1@9IGy1)RZ>d=4y literal 0 HcmV?d00001 From 5aae50449866adac65bbcf5ce70c64f8fe419030 Mon Sep 17 00:00:00 2001 From: Ryan Berger Date: Sun, 26 Apr 2026 22:33:47 -0700 Subject: [PATCH 7/8] fix: syncing and provision benchmark We now sync two tiffs rather than using COGs so use the `tvs` and `longest_lines` directories There was also an issue with the benchmark running at provision time because the job doesn't cd into the correct directory to run the benchmark. --- crates/tasks/src/atlas/tile_job.rs | 24 ++++++++---------------- scripts/cloud_init_ubuntu22.bash | 2 +- scripts/google_cloud.bash | 2 +- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/crates/tasks/src/atlas/tile_job.rs b/crates/tasks/src/atlas/tile_job.rs index 8368aed..0dbd808 100644 --- a/crates/tasks/src/atlas/tile_job.rs +++ b/crates/tasks/src/atlas/tile_job.rs @@ -201,7 +201,7 @@ impl TileRunner<'_> { } if !self.job.config.is_local_run() { - self.s3_put_longest_lines_cog(&self.job.tile.cog_filename()) + self.s3_put_longest_lines_tiff() .await?; } @@ -213,10 +213,10 @@ impl TileRunner<'_> { /// There isn't a huge difference between this and the post-processed one, but it's a shame /// to have to recompute the entire planet just to get hold of this. async fn s3_put_raw_tvs_tiff(&self) -> Result<()> { - let tvs_tiff = self.job.tile.cog_filename(); - let source = format!("{}/tmp/plain.tif", self.job_directory); + let filename = self.job.tile.cog_filename(); + let source = format!("{}/total_surfaces.tiff", self.job_directory); let destination = format!( - "s3://viewview/runs/{}/raw/{tvs_tiff}", + "s3://viewview/runs/{}/tvs/{filename}", self.job.config.run_id ); @@ -226,21 +226,13 @@ impl TileRunner<'_> { } /// Sync a longest lines COG to our S3 bucket. - async fn s3_put_longest_lines_cog(&self, filename: &str) -> Result<()> { - let source_cogs = self - .job - .config - .longest_lines_cogs - .clone() - .unwrap_or_else(|| LONGEST_LINES_DIRECTORY.into()) - .join(filename) - .display() - .to_string(); + async fn s3_put_longest_lines_tiff(&self) -> Result<()> { - let source = format!("{}/{source_cogs}", self.job_directory); + let filename = self.job.tile.cog_filename(); + let source = format!("{}/longest_lines.tiff", self.job_directory); let destination = format!( - "s3://viewview/runs/{}/longest_lines_cogs/{filename}", + "s3://viewview/runs/{}/longest_lines/{filename}", self.job.config.run_id ); diff --git a/scripts/cloud_init_ubuntu22.bash b/scripts/cloud_init_ubuntu22.bash index bd1d0d3..9eb2a37 100644 --- a/scripts/cloud_init_ubuntu22.bash +++ b/scripts/cloud_init_ubuntu22.bash @@ -31,7 +31,7 @@ function cloud_init_ubuntu22 { ssh "$address" "\ set -euo pipefail source ~/.cargo/env - ./benchmarks/run.sh cpu + cd ~/tvs && ./benchmarks/run.sh cpu cd ~/tvs && RUSTFLAGS='-Ctarget-cpu=native' cargo build --release rm -r ~/.rustup/ rm -r ~/.cargo/ diff --git a/scripts/google_cloud.bash b/scripts/google_cloud.bash index 9f4131a..1503255 100644 --- a/scripts/google_cloud.bash +++ b/scripts/google_cloud.bash @@ -3,7 +3,7 @@ function spin_google_cloud { gcloud compute instances create "$1" --format="json" \ - --zone=us-central1-b \ + --zone=us-central1-a \ --machine-type=h4d-standard-192 \ --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \ --metadata=startup-script="sudo rm -r /usr/lib/google-cloud-sdk/",ssh-keys=atlas:"$2" \ From 65520a9127093fd7b9992315318059f70d5db1be Mon Sep 17 00:00:00 2001 From: Thomas Buckley-Houston Date: Tue, 28 Apr 2026 17:43:46 +0100 Subject: [PATCH 8/8] Post-process longest lines COG during Atlas compute --- README.md | 7 +- crates/tasks/src/atlas/longest_lines/index.rs | 15 ++-- crates/tasks/src/atlas/tile_job.rs | 70 ++++++++++++++----- crates/tasks/src/config.rs | 4 -- crates/tasks/src/tile.rs | 2 +- 5 files changed, 59 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 256962a..6d1a555 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ RUST_LOG=info,axum=trace,apalis=trace,tasks=trace \ ``` Add tile jobs: +(tiles are automatically pulled down from s3://viewview/stitched) ``` # Saving data locally, useful for development: @@ -132,8 +133,7 @@ RUST_LOG=trace cargo run --bin tasks -- atlas run \ --centre -4.549624919891357,47.44954299926758 \ --amount 1 \ --skip 10 \ - --tvs-executable ../total-viewsheds/target/release/total-viewsheds \ - --longest-lines-cogs website/public/longest_lines + --tvs-executable ../total-viewsheds/target/release/total-viewsheds # Saving data on remote machines and S3, used for production: RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ @@ -141,8 +141,7 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ --run-id 0.1 \ --master website/public/tiles.csv \ --centre -13.949958801269531,57.94995880126953 \ - --tvs-executable /root/tvs/target/release/total-viewsheds \ - --longest-lines-cogs output/longest_lines + --tvs-executable /root/tvs/target/release/total-viewsheds ``` Atlas doesn't run the following commands, you'll want to manually run them after a diff --git a/crates/tasks/src/atlas/longest_lines/index.rs b/crates/tasks/src/atlas/longest_lines/index.rs index 115e31b..5541ca3 100644 --- a/crates/tasks/src/atlas/longest_lines/index.rs +++ b/crates/tasks/src/atlas/longest_lines/index.rs @@ -12,19 +12,12 @@ pub async fn compile() -> Result<()> { color_eyre::eyre::bail!("Can't save longest lines when there's no current run config."); }; + let index_path = "longest_lines_index.txt"; let tiles = crate::atlas::db::get_completed_tiles().await?; - let default_directory = std::path::Path::new(crate::atlas::tile_job::WORKING_DIRECTORY) - .join(crate::atlas::tile_job::LONGEST_LINES_DIRECTORY); - - let index_path = config - .clone() - .longest_lines_cogs - .unwrap_or(default_directory) - .join("index.txt"); let mut index = Vec::new(); for tile in tiles { - let filename = tile.cog_filename(); + let filename = tile.canonical_filename(); let line = format!("{filename} {}", tile.width); index.push(line); } @@ -34,7 +27,7 @@ pub async fn compile() -> Result<()> { index.len(), index_path ); - std::fs::write(index_path.clone(), index.join("\n"))?; + std::fs::write(index_path, index.join("\n"))?; if !config.is_local_run() { let destination = format!( @@ -42,7 +35,7 @@ pub async fn compile() -> Result<()> { config.run_id ); crate::atlas::machines::local::Machine::connection() - .sync_file_to_s3(&index_path.display().to_string(), &destination) + .sync_file_to_s3(index_path, &destination) .await?; } diff --git a/crates/tasks/src/atlas/tile_job.rs b/crates/tasks/src/atlas/tile_job.rs index 0dbd808..3c84d86 100644 --- a/crates/tasks/src/atlas/tile_job.rs +++ b/crates/tasks/src/atlas/tile_job.rs @@ -12,8 +12,8 @@ use tokio::sync::Mutex; /// The directory where all our viewview input/output goes. pub const WORKING_DIRECTORY: &str = "work"; -/// The default directory where all the longest lines COGs live. -pub const LONGEST_LINES_DIRECTORY: &str = "longest_lines"; +/// Filename for longest lines COG. +const LONGEST_LINES_COG: &str = "longest_lines.cog.tiff"; #[derive(Debug, serde::Serialize, serde::Deserialize)] /// A worker job that processes a tile. @@ -196,28 +196,61 @@ impl TileRunner<'_> { /// Process the assets needed to display the output on the website. async fn assets(&self) -> Result<()> { + self.make_longest_lines_cog().await?; + if !self.job.config.is_local_run() { + self.s3_put_longest_lines_cog().await?; self.s3_put_raw_tvs_tiff().await?; } - if !self.job.config.is_local_run() { - self.s3_put_longest_lines_tiff() - .await?; - } + Ok(()) + } + + /// It needs to be a COG because it's queried from the browser. + async fn make_longest_lines_cog(&self) -> Result<()> { + let plain_tif = format!("{}/longest_lines.tiff", self.job_directory); + let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG); + + let args = vec![ + "-of", + "COG", + // Smallest valid size (cos we're just querying for single raster points) + "-co", + "BLOCKSIZE=128", + "-co", + "RESAMPLING=NEAREST", + "-co", + "OVERVIEWS=NONE", + "-co", + "COMPRESS=DEFLATE", + "-co", + "PREDICTOR=3", + &plain_tif, + &cog, + ]; + + self.machine + .command(crate::atlas::machines::connection::Command { + executable: "/usr/bin/gdal_translate".into(), + args, + env: vec![], + ..Default::default() + }) + .await?; Ok(()) } /// Sync the raw, pre-projected finished heatmap for the tile to our S3 bucket. /// - /// There isn't a huge difference between this and the post-processed one, but it's a shame - /// to have to recompute the entire planet just to get hold of this. + /// It's tempting to do the full post-processing here, but it's a lossy step, so any mistake + /// and we can't get back the data. async fn s3_put_raw_tvs_tiff(&self) -> Result<()> { - let filename = self.job.tile.cog_filename(); let source = format!("{}/total_surfaces.tiff", self.job_directory); let destination = format!( - "s3://viewview/runs/{}/tvs/{filename}", - self.job.config.run_id + "s3://viewview/runs/{}/raw/{}", + self.job.config.run_id, + self.job.tile.canonical_filename() ); self.machine.sync_file_to_s3(&source, &destination).await?; @@ -226,17 +259,16 @@ impl TileRunner<'_> { } /// Sync a longest lines COG to our S3 bucket. - async fn s3_put_longest_lines_tiff(&self) -> Result<()> { - - let filename = self.job.tile.cog_filename(); - let source = format!("{}/longest_lines.tiff", self.job_directory); - + /// It needs to be a COG because it's queried from the browser. + async fn s3_put_longest_lines_cog(&self) -> Result<()> { + let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG); let destination = format!( - "s3://viewview/runs/{}/longest_lines/{filename}", - self.job.config.run_id + "s3://viewview/runs/{}/longest_lines_cogs/{}", + self.job.config.run_id, + self.job.tile.canonical_filename() ); - self.machine.sync_file_to_s3(&source, &destination).await?; + self.machine.sync_file_to_s3(&cog, &destination).await?; Ok(()) } diff --git a/crates/tasks/src/config.rs b/crates/tasks/src/config.rs index 8e3e4dd..53fe632 100644 --- a/crates/tasks/src/config.rs +++ b/crates/tasks/src/config.rs @@ -186,10 +186,6 @@ pub struct Atlas { #[arg(long, value_name = "TVS executable")] pub tvs_executable: std::path::PathBuf, - /// Where to load/save longest lines COGs. - #[arg(long, value_name = "Longest lines COGs directory")] - pub longest_lines_cogs: Option, - /// Where to run the computations, locally or on a cloud provider. #[arg( long, diff --git a/crates/tasks/src/tile.rs b/crates/tasks/src/tile.rs index 9dd9998..735446a 100644 --- a/crates/tasks/src/tile.rs +++ b/crates/tasks/src/tile.rs @@ -112,7 +112,7 @@ impl Tile { } /// Canonical filename for the tile. - pub fn cog_filename(&self) -> String { + pub fn canonical_filename(&self) -> String { format!("{}_{}.tiff", self.centre.0.x, self.centre.0.y) } }