diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e8c8d18..0b11ae5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,17 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Linux system dependencies + run: | + sudo apt-get update -y + sudo apt-get install libgdal-dev gdal-bin + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.94.1 + components: clippy,rustfmt - name: Install pnpm uses: pnpm/action-setup@v4 with: @@ -60,3 +71,8 @@ jobs: working-directory: website run: node_modules/.bin/biome check + - name: Lint + run: cargo clippy -- --deny warnings + - name: Formatting + run: cargo fmt --check + diff --git a/README.md b/README.md index 747107f..6d1a555 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # A View Of All Views +![Screenshot of app](screenshot.webp) + This repo is for all the supporting code used to find and display the longest line of sight on the planet. The main viewshed algorithm is another repo https://github.com/AllTheLines/CacheTVS @@ -69,26 +71,33 @@ So can we strike an optimal balance? This is what the `Packer` in this repo trie Creates arbitrary tiles out of the global DEM data. +Stitch one: ``` cargo run --bin tasks -- stitch \ --dems /publicish/dems \ --centre -3.049208402633667,53.24937438964844 \ --width 366610.1875 ``` +(optionally takes `--scale` argument) + +Stitch all from a folder: +``` +cargo run --bin tasks -- atlas stitch-all \ + --tmp-directory /mnt/disks/data/tmp \ + --dems /mnt/disks/data/srtm/ \ + --master website/public/tiles.csv +``` +(optionally takes `--scale` argument) ## Calculate Total Viewsheds Using https://github.com/AllTheLines/CacheTVS -Outputs `.bt` heatmap. +Outputs `total_surfaces.tiff` and `longest_lines.tiff`. ## Prepare For Cloud ``` -# Note that all these depend on a `./output` path existing. -./ctl.sh prepare_for_cloud ../total-viewsheds/output/total_surfaces.bt -./ctl.sh prepare_for_cloud ../total-viewsheds/output/longest_lines.bt - ./ctl.sh make_pmtiles latest website/public/world.pmtiles ``` @@ -113,6 +122,7 @@ RUST_LOG=info,axum=trace,apalis=trace,tasks=trace \ ``` Add tile jobs: +(tiles are automatically pulled down from s3://viewview/stitched) ``` # Saving data locally, useful for development: @@ -123,8 +133,7 @@ RUST_LOG=trace cargo run --bin tasks -- atlas run \ --centre -4.549624919891357,47.44954299926758 \ --amount 1 \ --skip 10 \ - --tvs-executable ../total-viewsheds/target/release/total-viewsheds \ - --longest-lines-cogs website/public/longest_lines + --tvs-executable ../total-viewsheds/target/release/total-viewsheds # Saving data on remote machines and S3, used for production: RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ @@ -132,21 +141,22 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ --run-id 0.1 \ --master website/public/tiles.csv \ --centre -13.949958801269531,57.94995880126953 \ - --tvs-executable /root/tvs/target/release/total-viewsheds \ - --longest-lines-cogs output/longest_lines + --tvs-executable /root/tvs/target/release/total-viewsheds ``` Atlas doesn't run the following commands, you'll want to manually run them after a bunch of tiles have been processed: -* Create the gigantic (10s of GBs) global `.pmtile` that contains the TVS heatmap for the entire planet. +* Create the gigantic (~500GB) global `.pmtile` that contains the TVS heatmap for the entire planet. ``` # This requires a machine with a lot of disk, RAM and CPU. As of writing, I'd recommend # 3TB disk, >150GB RAM and at least 48 cores. # Replace `latest` with `local` to skip syncing files to S3. -./ctl.sh make_pmtiles latest work/world.pmtiles +./ctl.sh make_pmtiles latest work/world.pmtiles 100 ``` +Note that this process creates a _temporary_ version of the `.pmtile`, and then copies that to its final destination. So you need at least 1.2TB of disk for a world PMTile. And then of course you need space to download the raw .tiffs, and space to save the post-processed versions of those .tiffs. So at least 2TB is a good idea. + * Create an index of all the COG (optimised GeoTiff) files that contain all the longest lines of sight for every point on the planet. Should only take seconds to run. ``` RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas longest-lines-index @@ -158,7 +168,7 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas longest-lines-index RUST_LOG=off,tasks=trace cargo run --release --bin tasks -- \ atlas longest-lines-overviews \ - --tiffs work/longest_lines + --tiffs work/longest_lines --run-id 0.1 ``` ## Static Site Website diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 5daf4c0..dd8a645 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -8,4 +8,3 @@ pub mod metadata; pub mod projector; pub mod utils; - diff --git a/crates/shared/src/projector.rs b/crates/shared/src/projector.rs index b28d354..0298f89 100644 --- a/crates/shared/src/projector.rs +++ b/crates/shared/src/projector.rs @@ -3,7 +3,7 @@ use color_eyre::{Result, eyre::ContextCompat as _}; /// The radius of the planet in kilometers. -pub const EARTH_RADIUS: f32 = 6371.0; +pub const EARTH_RADIUS: f32 = 6378.137; /// A longtitude/latitude coordinate. #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Default)] diff --git a/crates/tasks/src/atlas/daemon.rs b/crates/tasks/src/atlas/daemon.rs index 6e3367f..36fef01 100644 --- a/crates/tasks/src/atlas/daemon.rs +++ b/crates/tasks/src/atlas/daemon.rs @@ -7,8 +7,7 @@ use apalis::{layers::WorkerBuilderExt as _, prelude::Layer as _}; /// The URL for the worker web UI. const WEB_UI_HOST: &str = "localhost:3003"; - -/// Start the Atlas daemon +/// Start the Atlas daemon. pub async fn start_all( _: &crate::config::Worker, broadcaster: std::sync::Arc>, @@ -53,10 +52,8 @@ async fn start_existing_machines() -> Result<()> { ); for job in jobs { - crate::atlas::machines::new_machine_job::new_machine_handler( - job.machine.as_ref().clone(), - ) - .await?; + crate::atlas::machines::new_machine_job::new_machine_handler(job.machine.as_ref().clone()) + .await?; } Ok(()) diff --git a/crates/tasks/src/atlas/longest_lines/grided.rs b/crates/tasks/src/atlas/longest_lines/grided.rs index ea85c4c..6c6f9a2 100644 --- a/crates/tasks/src/atlas/longest_lines/grided.rs +++ b/crates/tasks/src/atlas/longest_lines/grided.rs @@ -8,9 +8,9 @@ use tokio::io::AsyncWriteExt as _; #[repr(C)] #[derive(Copy, Clone, Debug, bytemuck::Zeroable, bytemuck::Pod)] pub struct Grided { - /// Longtitude + /// Longtitude. pub lon: f32, - /// Latitude + /// Latitude. pub lat: f32, /// The distance of the line of sight in meters. pub distance: u32, diff --git a/crates/tasks/src/atlas/longest_lines/index.rs b/crates/tasks/src/atlas/longest_lines/index.rs index a5da9e4..5541ca3 100644 --- a/crates/tasks/src/atlas/longest_lines/index.rs +++ b/crates/tasks/src/atlas/longest_lines/index.rs @@ -12,23 +12,13 @@ pub async fn compile() -> Result<()> { color_eyre::eyre::bail!("Can't save longest lines when there's no current run config."); }; + let index_path = "longest_lines_index.txt"; let tiles = crate::atlas::db::get_completed_tiles().await?; - let default_directory = std::path::Path::new(crate::atlas::tile_job::WORKING_DIRECTORY) - .join(crate::atlas::tile_job::LONGEST_LINES_DIRECTORY); - - let index_path = config - .clone() - .longest_lines_cogs - .unwrap_or(default_directory) - .join("index.txt"); let mut index = Vec::new(); for tile in tiles { - let filename = tile.cog_filename(); - let line = format!( - "{filename} {}", - tile.width * crate::atlas::tile_job::DEM_SCALE - ); + let filename = tile.canonical_filename(); + let line = format!("{filename} {}", tile.width); index.push(line); } @@ -37,7 +27,7 @@ pub async fn compile() -> Result<()> { index.len(), index_path ); - std::fs::write(index_path.clone(), index.join("\n"))?; + std::fs::write(index_path, index.join("\n"))?; if !config.is_local_run() { let destination = format!( @@ -45,7 +35,7 @@ pub async fn compile() -> Result<()> { config.run_id ); crate::atlas::machines::local::Machine::connection() - .sync_file_to_s3(&index_path.display().to_string(), &destination) + .sync_file_to_s3(index_path, &destination) .await?; } diff --git a/crates/tasks/src/atlas/longest_lines/overview.rs b/crates/tasks/src/atlas/longest_lines/overview.rs index f1c4b1b..918a988 100644 --- a/crates/tasks/src/atlas/longest_lines/overview.rs +++ b/crates/tasks/src/atlas/longest_lines/overview.rs @@ -23,18 +23,18 @@ struct LongestLine { /// A hash where each key is an H3 grid cell and its value is the longest line of sight within /// that grid cell. -type HashMap = std::collections::HashMap; +type LongestInGrids = std::collections::HashMap; -/// Keep track of the current longest line in each H3 grid. -type StateHash = Arc>; +/// Keep track of work per tile. +type StateHash = Arc>>; /// Representation of a longest lines COG file. struct Tile { - /// The AEQD to lon/lat coordinate converter + /// The AEQD to lon/lat coordinate converter. projector: shared::projector::Convert, - /// The raw point data for the tile + /// The raw point data for the tile. buffer: gdal::raster::Buffer, - /// The width of the tile in points + /// The width of the tile in points. width: usize, /// Offset to the centre of the tile in points. offset: f64, @@ -42,7 +42,7 @@ struct Tile { impl Tile { /// Entrypoint. - fn process(path: &std::path::Path) -> Result { + fn process(path: &std::path::Path) -> Result { let centre = Self::parse_centre_coords(path)?; let buffer = Self::load(path)?; let width = buffer.width(); @@ -105,10 +105,10 @@ impl Tile { /// Iterate through every longest line in COG, associate it with a H3 grid, check to see if /// it's the longest in the grid and set record it if so. - fn find_longest_lines(&self) -> Result { + fn find_longest_lines(&self) -> Result { tracing::trace!("Looping through {} points", self.buffer.len()); - let mut local = HashMap::new(); + let mut local = LongestInGrids::new(); for (index, &value) in self.buffer.data().iter().enumerate() { let coord = self.index_to_coord(index); let lonlat = self.coord_to_lonlat(coord)?; @@ -192,45 +192,66 @@ pub async fn run(config: &crate::config::LongestLinesOverviews) -> Result<()> { let world_clone = Arc::clone(&world); let jobs_clone = Arc::clone(&jobs); let completed_jobs = Arc::clone(&completed_jobs_master); - let handle = tokio::task::spawn(async move { - tracing::debug!("Spawning worker {worker}"); - - loop { - let job = jobs_clone.write().await.pop(); - match job { - Some(tiff) => { - let result: Result<()> = async { - let local = Tile::process(&tiff)?; - update_state(&world_clone, local).await?; - completed_jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - tracing::info!( - "{}/{total_jobs} completed jobs", - completed_jobs.load(std::sync::atomic::Ordering::Relaxed) - ); - Ok(()) + + tracing::debug!("Spawning worker {worker}"); + let handle = std::thread::spawn(move || { + #[expect(clippy::expect_used, reason = "I'm lazy")] + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("Couldn't build Tokio runtime in current thread"); + + runtime.block_on(async { + loop { + let job = jobs_clone.write().await.pop(); + match job { + Some(tiff) => { + let filename = tiff.display().to_string(); + let result: Result<()> = async { + let local = Tile::process(&tiff)?; + world_clone.write().await.insert(filename.clone(), local); + let current_world = world_clone.read().await.clone(); + std::thread::spawn(move || compile_world(¤t_world)); + completed_jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + tracing::info!( + "{}/{total_jobs} completed jobs", + completed_jobs.load(std::sync::atomic::Ordering::Relaxed) + ); + Ok(()) + } + .await; + if let Err(error) = result { + tracing::error!( + "Error running longest lines job for {filename}: {error}" + ); + #[expect(clippy::exit, reason = "Everything needs to work!")] + std::process::exit(1); + } } - .await; - if let Err(error) = result { - tracing::error!("Error running longest lines job: {error}"); - #[expect(clippy::exit, reason = "Everything needs to work!")] - std::process::exit(1); + None => { + tracing::debug!("Worker {worker} shutting down"); + break; } } - None => { - tracing::debug!("Worker {worker} shutting down"); - break; - } } - } + }); }); handles.push(handle); } - for result in futures::future::join_all(handles).await { - result?; + for handle in handles { + #[expect(clippy::expect_used, reason = "I'm lazy")] + handle.join().expect("Error joining thread"); } + let grided = compile_world(&world.read().await.clone())?; + super::grided::write( + format!("./output/{LONGEST_LINES_GRIDED_FILENAME}",).into(), + &grided, + ) + .await?; + sync_to_s3( format!("./output/{LONGEST_LINES_GRIDED_FILENAME}").into(), &config.run_id, @@ -240,17 +261,22 @@ pub async fn run(config: &crate::config::LongestLinesOverviews) -> Result<()> { Ok(()) } -/// Take all the newly found longest lines in a COG and check to see if they're longer than any of -/// the currently recorded global ones. -async fn update_state(world: &StateHash, local: HashMap) -> Result<()> { +/// Take all the per-tile grided longest lines and compile a global canonical grid of longest lines. +fn compile_world( + world: &std::collections::HashMap, +) -> Result> { + let mut master = LongestInGrids::new(); + #[expect(clippy::iter_over_hash_type, reason = "We don't mind about ordering")] - for (cell, line) in local { - if let Some(existing) = world.write().await.get_mut(&cell) { - if line.packed.distance() > existing.packed.distance() { - *existing = line; + for grid_for_tile in world.values() { + for (cell, line) in grid_for_tile { + if let Some(existing) = master.get_mut(cell) { + if line.packed.distance() > existing.packed.distance() { + *existing = *line; + } + } else { + master.insert(*cell, *line); } - } else { - world.write().await.insert(cell, line); } } @@ -261,9 +287,10 @@ async fn update_state(world: &StateHash, local: HashMap) -> Result<()> { }; let mut grided = Vec::new(); - let count = world.read().await.len(); + let count = master.len(); + #[expect(clippy::iter_over_hash_type, reason = "We don't mind about ordering")] - for entry in world.read().await.values() { + for entry in master.values() { if entry.packed.distance() > longest.packed.distance() { longest = *entry; } @@ -296,13 +323,7 @@ async fn update_state(world: &StateHash, local: HashMap) -> Result<()> { ); }; - super::grided::write( - format!("./output/{LONGEST_LINES_GRIDED_FILENAME}",).into(), - &grided, - ) - .await?; - - Ok(()) + Ok(grided) } /// Find all the `.tiff`s in a given directory. diff --git a/crates/tasks/src/atlas/machines/connection.rs b/crates/tasks/src/atlas/machines/connection.rs index 50c8e45..40256a2 100644 --- a/crates/tasks/src/atlas/machines/connection.rs +++ b/crates/tasks/src/atlas/machines/connection.rs @@ -20,7 +20,7 @@ pub struct Command<'command> { pub args: Vec<&'command str>, /// Environment variables to pass to the command process. pub env: Vec<(&'command str, &'command str)>, - /// The directory in which to run the command + /// The directory in which to run the command. pub current_dir: Option, } diff --git a/crates/tasks/src/atlas/machines/digital_ocean.rs b/crates/tasks/src/atlas/machines/digital_ocean.rs index 3fdefe9..a374524 100644 --- a/crates/tasks/src/atlas/machines/digital_ocean.rs +++ b/crates/tasks/src/atlas/machines/digital_ocean.rs @@ -1,4 +1,4 @@ -//! Managing Digital Ocean resources +//! Managing Digital Ocean resources. use color_eyre::Result; diff --git a/crates/tasks/src/atlas/machines/google_cloud.rs b/crates/tasks/src/atlas/machines/google_cloud.rs index fdc33fa..0f721e7 100644 --- a/crates/tasks/src/atlas/machines/google_cloud.rs +++ b/crates/tasks/src/atlas/machines/google_cloud.rs @@ -1,16 +1,15 @@ -//! `google_cloud` is a Google Cloud +//! `google_cloud` is a Google Cloud. use color_eyre::eyre::Result; -use std::net::{IpAddr}; -use std::time::{SystemTime}; +use std::net::IpAddr; +use std::time::SystemTime; - -/// Machine implements the `machine::Machine` trait for privisioning a Google Cloud machine +/// Machine implements the `machine::Machine` trait for privisioning a Google Cloud machine. pub struct Machine; impl Machine { /// provision runs the whole provisioning workflow for a Google Cloud machine, - /// returning the IP address of the new machine + /// returning the IP address of the new machine. async fn provision(name: &str, ssh_key_id: &str) -> Result { let command = super::connection::Command { executable: "./ctl.sh".into(), @@ -28,13 +27,18 @@ impl Machine { /// /// Google Cloud frequently hands out the same IP to different runs creating /// issues with host checking. Instead of infecting all other ssh commands with key checking, - /// it is much easier just to clear that IP out from `known_hosts` + /// it is much easier just to clear that IP out from `known_hosts`. async fn wait_for_machine_to_boot(ip_address: IpAddr) -> Result<()> { let ip_string = ip_address.to_string(); let delete_ip = super::connection::Command { executable: "ssh-keygen".into(), - args: vec!["-f", "/home/ryan/.ssh/known_hosts", "-R", ip_string.as_str()], + args: vec![ + "-f", + "/home/ryan/.ssh/known_hosts", + "-R", + ip_string.as_str(), + ], ..Default::default() }; super::local::Machine::command(delete_ip.clone()).await?; @@ -61,7 +65,7 @@ impl Machine { color_eyre::eyre::bail!("Timed out waiting for machine {ip_string} to boot."); } - /// `init` provisions a `GoogleCloud` Debian machine with the Ubuntu startup script + /// `init` provisions a `GoogleCloud` Debian machine with the Ubuntu startup script. async fn init(ip_address: IpAddr) -> Result { let connect = format!("atlas@{ip_address}"); diff --git a/crates/tasks/src/atlas/machines/new_machine_job.rs b/crates/tasks/src/atlas/machines/new_machine_job.rs index d73711b..37991d2 100644 --- a/crates/tasks/src/atlas/machines/new_machine_job.rs +++ b/crates/tasks/src/atlas/machines/new_machine_job.rs @@ -21,9 +21,7 @@ impl NewMachineJob { } /// The callback to run when a new machine job is added. -pub async fn new_machine_handler( - job: super::new_machine_job::NewMachineJob, -) -> Result<()> { +pub async fn new_machine_handler(job: super::new_machine_job::NewMachineJob) -> Result<()> { let machine_task_id = get_machine_task_id_hack(job.clone()).await?; tokio::spawn(async move { @@ -36,7 +34,7 @@ pub async fn new_machine_handler( } /// Get the internal ID of the job. Just a hack until this is fixed: -/// +/// . async fn get_machine_task_id_hack( job_to_find: super::new_machine_job::NewMachineJob, ) -> Result { diff --git a/crates/tasks/src/atlas/machines/vultr.rs b/crates/tasks/src/atlas/machines/vultr.rs index f7da7be..36543da 100644 --- a/crates/tasks/src/atlas/machines/vultr.rs +++ b/crates/tasks/src/atlas/machines/vultr.rs @@ -1,4 +1,4 @@ -//! Managing Vultr resources +//! Managing Vultr resources. use color_eyre::Result; diff --git a/crates/tasks/src/atlas/machines/worker.rs b/crates/tasks/src/atlas/machines/worker.rs index f8b9f87..af62c3c 100644 --- a/crates/tasks/src/atlas/machines/worker.rs +++ b/crates/tasks/src/atlas/machines/worker.rs @@ -62,7 +62,8 @@ pub async fn tile_processor( }; // clear out the previous run's state - state.daemon + state + .daemon .command(crate::atlas::machines::connection::Command { executable: "rm".into(), args: vec!["-rf", WORKING_DIRECTORY], @@ -74,7 +75,6 @@ pub async fn tile_processor( // Computation concurrency is effectively 1 due to mutex locking in the tile job. let worker_concurrency = 2; - WorkerBuilder::new(tile_worker_name.clone()) .backend(tile_store) .data(Arc::new(state)) diff --git a/crates/tasks/src/atlas/run.rs b/crates/tasks/src/atlas/run.rs index 94620e6..ac2ab96 100644 --- a/crates/tasks/src/atlas/run.rs +++ b/crates/tasks/src/atlas/run.rs @@ -4,10 +4,9 @@ //! 2. Get tiles around a point that haven't been crunched yet //! 1. Stitch the relevant tile //! 2. TVS the tile -//! 3. Run `.ctl.sh prepare_for_cloud` for `total_surfaces.bt` and `longest_lines.bt`. //! //! These can and probably should be run manually after a bunch of tiles have been processed. -//! 4. Create the new `world.pmtile`: `./ctl.sh make_pmtiles latest website/public/world.pmtiles` +//! 3. Create the new `world.pmtile`: `./ctl.sh make_pmtiles latest website/public/world.pmtiles`. use color_eyre::{Result, eyre::ContextCompat as _}; @@ -113,6 +112,11 @@ impl Atlas { tile: master_tile.data, }; + #[expect( + clippy::as_conversions, + clippy::cast_possible_truncation, + reason = "Truncation is unlikely to happen™" + )] // map the priority of the tile width (10,000->900,000) to an integer 1-9 // and then invert the priority so that the smallest go first let priority = -((tile_args.tile.width / 100_000.0f32) as i32); diff --git a/crates/tasks/src/atlas/stitch_all.rs b/crates/tasks/src/atlas/stitch_all.rs index 9a6186e..8540177 100644 --- a/crates/tasks/src/atlas/stitch_all.rs +++ b/crates/tasks/src/atlas/stitch_all.rs @@ -11,7 +11,7 @@ use color_eyre::Result; pub struct StitchJob { /// Config from the CLI. pub config: crate::config::StitchAll, - /// The tile to process + /// The tile to process. pub tile: crate::tile::Tile, } @@ -55,18 +55,34 @@ pub async fn process(job: StitchJob) -> Result<()> { let dems_path = job.config.dems.display().to_string(); let centre = format!("{},{}", job.tile.centre.0.x, job.tile.centre.0.y); let width = job.tile.width.to_string(); + let tmp_directory = job.config.tmp_directory.display().to_string(); + let mut args = [ + "stitch", + "--dems", + &dems_path, + "--centre", + ¢re, + "--width", + &width, + "--output-dir", + &tmp_directory, + ] + .iter() + .map(std::string::ToString::to_string) + .collect::>(); + if let Some(scale) = job.config.scale { + args.extend(vec!["--scale".to_owned(), scale.to_string()]); + } let command = super::machines::connection::Command { - executable: "target/debug/tasks".into(), - args: vec![ - "stitch", "--dems", &dems_path, "--centre", ¢re, "--width", &width, - ], + executable: std::env::current_exe()?, + args: args.iter().map(std::string::String::as_str).collect(), ..Default::default() }; super::machines::local::Machine::command(command).await?; let stitch_tile_path = crate::stitch::canonical_filename(job.tile.centre.0.x, job.tile.centre.0.y); - let source = format!("output/{stitch_tile_path}"); + let source = format!("{tmp_directory}/{stitch_tile_path}"); let destination = format!("s3://viewview/stitched/{stitch_tile_path}"); let local = super::machines::local::Machine::connection(); local.sync_file_to_s3(&source, &destination).await?; diff --git a/crates/tasks/src/atlas/tile_job.rs b/crates/tasks/src/atlas/tile_job.rs index f839f38..3c84d86 100644 --- a/crates/tasks/src/atlas/tile_job.rs +++ b/crates/tasks/src/atlas/tile_job.rs @@ -2,31 +2,29 @@ use crate::atlas::machines::connection::Connection; use crate::config::RUN_ID_LOCAL; +use apalis::prelude::WorkerContext; use clap::ValueEnum as _; use color_eyre::{Result, eyre::ContextCompat as _}; use std::sync::Arc; use std::time::SystemTime; -use apalis::prelude::WorkerContext; use tokio::sync::Mutex; -/// The size of each point in the raw DEM data. -pub const DEM_SCALE: f32 = 100.0; - /// The directory where all our viewview input/output goes. pub const WORKING_DIRECTORY: &str = "work"; -/// The default directory where all the longest lines COGs live. -pub const LONGEST_LINES_DIRECTORY: &str = "longest_lines"; +/// Filename for longest lines COG. +const LONGEST_LINES_COG: &str = "longest_lines.cog.tiff"; #[derive(Debug, serde::Serialize, serde::Deserialize)] /// A worker job that processes a tile. pub struct TileJob { /// Config from the CLI. pub config: crate::config::Atlas, - /// The tile to process + /// The tile to process. pub tile: crate::tile::Tile, } +/// Data for running a single tile job. pub struct TileRunner<'mutex> { /// `mutex` is a borrowed mutex from the `Worker`. It makes sure that /// only a single `TileRunner` is running an L.o.S calculation. @@ -34,31 +32,30 @@ pub struct TileRunner<'mutex> { /// Details about this particular job. job: TileJob, /// The unique-per-job path prefix that is used to store files - /// in order to allow for unlimited concurrency + /// in order to allow for unlimited concurrency. job_directory: String, /// The connection to the machine where we run the compute parts of the job. machine: Arc, } -/// `TileState` is the necessary state that a `TileJob` worker needs to coordinate -/// between other workers. +/// Necessary state that a `TileJob` worker needs to coordinate between other workers. pub struct TileWorkerState { /// `mutex` makes sure that only one line of sight computation is happening at any given time. /// Any other part of the job can run at the same time, but running L.o.S is too computationally /// intensive to share - /// TODO: this might not have to be a mutex, but it is too time consuming to restart the jobs + /// TODO: this might not have to be a mutex, but it is too time consuming to restart the jobs. pub mutex: Arc>, - /// `daemon` holds an open ssh connection to the machine that all commands will be running on + /// `daemon` holds an open ssh connection to the machine that all commands will be running on. pub daemon: Arc, } /// `process_tile` does the work of processing a single tile. /// -/// It is a wrapper for `TileRunner` +/// It is a wrapper for `TileRunner`. pub async fn process_tile( job: TileJob, state: apalis::prelude::Data>, - ctx: WorkerContext + ctx: WorkerContext, ) -> Result<()> { tracing::info!("Processing tile: {:?}", job.tile); @@ -83,10 +80,10 @@ pub async fn process_tile( }; let result = runner.run().await; - if let Err(_) = result { + if result.is_err() { tracing::info!("shutting down worker {}", ctx.name()); ctx.stop()?; - return result + return result; } if cleanup { @@ -98,11 +95,11 @@ pub async fn process_tile( impl TileRunner<'_> { /// `run` sets up all directories, downloads necessary files, computes a L.o.S - /// and then does post-processing and uploads + /// and then does post-processing and uploads. async fn run(&self) -> Result<()> { self.ensure_directories().await?; - let bt_filepath = self.download_bt_file().await?; + let bt_filepath = self.download_stitched_geotiff().await?; self.compute(&bt_filepath).await?; @@ -143,11 +140,11 @@ impl TileRunner<'_> { // TODO: Make this its own job so it can be parallelised. /// Download the packer-found, pre-stitched `.bt` DEM tile data. - async fn download_bt_file(&self) -> Result { - let bt_filename = + async fn download_stitched_geotiff(&self) -> Result { + let filename = crate::stitch::canonical_filename(self.job.tile.centre.0.x, self.job.tile.centre.0.y); - let from = format!("s3://viewview/stitched/{bt_filename}"); - let to = format!("{}/{bt_filename}", self.job_directory); + let from = format!("s3://viewview/stitched/{filename}"); + let to = format!("{}/{filename}", self.job_directory); self.machine.sync_file_from_s3(&from, &to).await?; Ok(to) @@ -158,7 +155,6 @@ impl TileRunner<'_> { let _token = self.mutex.lock().await; let threads_as_string; - let scale = format!("{DEM_SCALE}"); let backend = self .job .config @@ -171,8 +167,6 @@ impl TileRunner<'_> { &bt_filepath, "--output-dir", &self.job_directory, - "--scale", - &scale, "--disable-image-render", "--backend", backend.get_name(), @@ -202,39 +196,44 @@ impl TileRunner<'_> { /// Process the assets needed to display the output on the website. async fn assets(&self) -> Result<()> { - self.prepare_for_cloud( - format!("{}/total_surfaces.bt", self.job_directory).as_str(), - &self.job.tile.cog_filename(), - ) - .await?; + self.make_longest_lines_cog().await?; if !self.job.config.is_local_run() { + self.s3_put_longest_lines_cog().await?; self.s3_put_raw_tvs_tiff().await?; } - self.prepare_for_cloud( - format!("{}/longest_lines.bt", self.job_directory).as_str(), - &self.job.tile.cog_filename(), - ) - .await?; - - if !self.job.config.is_local_run() { - self.s3_put_longest_lines_cog(&self.job.tile.cog_filename()) - .await?; - } - Ok(()) } - /// Prepare a processed tile for the website UI. - async fn prepare_for_cloud(&self, input: &str, output: &str) -> Result<()> { - let arguments = vec!["prepare_for_cloud", input, output]; + /// It needs to be a COG because it's queried from the browser. + async fn make_longest_lines_cog(&self) -> Result<()> { + let plain_tif = format!("{}/longest_lines.tiff", self.job_directory); + let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG); + + let args = vec![ + "-of", + "COG", + // Smallest valid size (cos we're just querying for single raster points) + "-co", + "BLOCKSIZE=128", + "-co", + "RESAMPLING=NEAREST", + "-co", + "OVERVIEWS=NONE", + "-co", + "COMPRESS=DEFLATE", + "-co", + "PREDICTOR=3", + &plain_tif, + &cog, + ]; self.machine .command(crate::atlas::machines::connection::Command { - executable: "./ctl.sh".into(), - args: arguments, - env: vec![("OUTPUT_DIR", &self.job_directory)], + executable: "/usr/bin/gdal_translate".into(), + args, + env: vec![], ..Default::default() }) .await?; @@ -244,14 +243,14 @@ impl TileRunner<'_> { /// Sync the raw, pre-projected finished heatmap for the tile to our S3 bucket. /// - /// There isn't a huge difference between this and the post-processed one, but it's a shame - /// to have to recompute the entire planet just to get hold of this. + /// It's tempting to do the full post-processing here, but it's a lossy step, so any mistake + /// and we can't get back the data. async fn s3_put_raw_tvs_tiff(&self) -> Result<()> { - let tvs_tiff = self.job.tile.cog_filename(); - let source = format!("{}/tmp/plain.tif", self.job_directory); + let source = format!("{}/total_surfaces.tiff", self.job_directory); let destination = format!( - "s3://viewview/runs/{}/raw/{tvs_tiff}", - self.job.config.run_id + "s3://viewview/runs/{}/raw/{}", + self.job.config.run_id, + self.job.tile.canonical_filename() ); self.machine.sync_file_to_s3(&source, &destination).await?; @@ -260,27 +259,17 @@ impl TileRunner<'_> { } /// Sync a longest lines COG to our S3 bucket. - async fn s3_put_longest_lines_cog(&self, filename: &str) -> Result<()> { - let source_cogs = self - .job - .config - .longest_lines_cogs - .clone() - .unwrap_or_else(|| LONGEST_LINES_DIRECTORY.into()) - .join(filename) - .display() - .to_string(); - - let source = format!("{}/{source_cogs}", self.job_directory); - + /// It needs to be a COG because it's queried from the browser. + async fn s3_put_longest_lines_cog(&self) -> Result<()> { + let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG); let destination = format!( - "s3://viewview/runs/{}/longest_lines_cogs/{filename}", - self.job.config.run_id + "s3://viewview/runs/{}/longest_lines_cogs/{}", + self.job.config.run_id, + self.job.tile.canonical_filename() ); - self.machine.sync_file_to_s3(&source, &destination).await?; + self.machine.sync_file_to_s3(&cog, &destination).await?; Ok(()) } } - diff --git a/crates/tasks/src/config.rs b/crates/tasks/src/config.rs index 5279190..53fe632 100644 --- a/crates/tasks/src/config.rs +++ b/crates/tasks/src/config.rs @@ -47,7 +47,7 @@ pub enum AtlasCommands { LongestLinesOverviews(LongestLinesOverviews), /// Output the current run's config. CurrentRunConfig(CurrentRunConfig), - /// Stitch the entire world's `.bt` files and save them to S3. + /// Stitch the entire world's `.tiff` files and save them to S3. StitchAll(StitchAll), } @@ -100,6 +100,19 @@ pub struct Stitch { /// The width of the tile in meters. #[arg(long, value_name = "Tile width")] pub width: f32, + + /// Where to save stitched Geotiffs. + #[arg( + long, + value_name = "Path to output directory", + default_value = "./output" + )] + pub output_dir: std::path::PathBuf, + + /// Override dynamic scaling. If not supplied the scale is based on the tile's distance from + /// the equator. + #[arg(long, value_name = "Raster scale")] + pub scale: Option, } /// `cargo run atlas stitch-all` arguments. @@ -113,9 +126,22 @@ pub struct StitchAll { #[arg(long, value_name = "Path to master tiles list")] pub master: std::path::PathBuf, + /// Where to save stitched Geotiffs before uploading to the bucket. + #[arg( + long, + value_name = "Path to temporary directory", + default_value = "./output" + )] + pub tmp_directory: std::path::PathBuf, + /// Number of CPUS to use. #[arg(long, value_name = "Number of cpus", default_value_t = number_of_cpus_on_machine())] pub num_cpus: usize, + + /// Override dynamic scaling. If not supplied the scale is based on the tile's distance from + /// the equator. + #[arg(long, value_name = "Raster scale")] + pub scale: Option, } /// Worker daemon to run Atlas jobs. @@ -160,10 +186,6 @@ pub struct Atlas { #[arg(long, value_name = "TVS executable")] pub tvs_executable: std::path::PathBuf, - /// Where to load/save longest lines COGs. - #[arg(long, value_name = "Longest lines COGs directory")] - pub longest_lines_cogs: Option, - /// Where to run the computations, locally or on a cloud provider. #[arg( long, diff --git a/crates/tasks/src/packer.rs b/crates/tasks/src/packer.rs index 270ac1f..d10a7cb 100644 --- a/crates/tasks/src/packer.rs +++ b/crates/tasks/src/packer.rs @@ -994,9 +994,9 @@ mod test { #[test] fn minimum_tile_size_for_elevation() { assert_eq!(Packer::minimum_tile_size_for_elevation(0), 36_000.0); - assert_eq!(Packer::minimum_tile_size_for_elevation(1000), 225_769.8); - assert_eq!(Packer::minimum_tile_size_for_elevation(3000), 391_075.44); - assert_eq!(Packer::minimum_tile_size_for_elevation(8000), 638_748.75); - assert_eq!(Packer::minimum_tile_size_for_elevation(8848), 671_772.3); + assert_eq!(Packer::minimum_tile_size_for_elevation(1000), 225_896.2); + assert_eq!(Packer::minimum_tile_size_for_elevation(3000), 391_294.38); + assert_eq!(Packer::minimum_tile_size_for_elevation(8000), 639_106.2); + assert_eq!(Packer::minimum_tile_size_for_elevation(8848), 672_148.2); } } diff --git a/crates/tasks/src/stitch.rs b/crates/tasks/src/stitch.rs index 06af1f5..d293fca 100644 --- a/crates/tasks/src/stitch.rs +++ b/crates/tasks/src/stitch.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use color_eyre::{Result, eyre::ContextCompat as _}; +use shared::projector::EARTH_RADIUS; /// A virtual DEM that represents _all_ the DEM data for the planet. const VIRTUAL_DEM_FILE: &str = "index.vrt"; @@ -12,6 +13,9 @@ const VIRTUAL_DEM_FILE: &str = "index.vrt"; /// How we mark points as containing no data. const NODATA_VALUE: &str = "-32768"; +/// The dimensions of an SRTM3 pixel at the equator. Pixels get smaller the further from the equator. +const SRTM3_RESOLUTION_AT_EQUATOR: f64 = 92.766_242_327_727_98f64; + /// Entrypoint. pub async fn make_tile( machine: &Arc, @@ -19,7 +23,6 @@ pub async fn make_tile( ) -> Result { build_virtual_dem(machine, config).await?; let filename = stitch(machine, config).await?; - set_centre_as_extent(machine, config, &filename).await?; Ok(filename) } @@ -88,7 +91,7 @@ fn find_all_hgts(config: &crate::config::Stitch) -> Result> { /// The canonical name for the stitched file. It's needed to be able to put and get the file from /// the S3 bucket. pub fn canonical_filename(lon: f64, lat: f64) -> String { - format!("{lon},{lat}.bt") + format!("{lon},{lat}.tiff") } /// Call `gdalwarp` to construct a new stitched tile. Data will also be interpolated to metric. @@ -96,14 +99,16 @@ async fn stitch( machine: &Arc, config: &crate::config::Stitch, ) -> Result { - let resolution = 100.0; - let resolution_string = resolution.to_string(); + let resolution = get_resolution(config); + let resolution_string = format!("{resolution:.12}"); + let aeqd = format!( "+proj=aeqd +lat_0={} +lon_0={} +units=m +datum=WGS84 +no_defs", config.centre.1, config.centre.0 ); let output = format!( - "./output/{}", + "{}/{}", + config.output_dir.display(), canonical_filename(config.centre.0, config.centre.1) ); let hgt_index = config.dems.join(VIRTUAL_DEM_FILE).display().to_string(); @@ -120,8 +125,8 @@ async fn stitch( (half_width * 2.0) / 3.0 ); - let min = format!("-{half_width}"); - let max = format!("{half_width}"); + let min = format!("-{half_width:.12}"); + let max = format!("{half_width:.12}"); let arguments = vec![ "-overwrite", "-dstnodata", @@ -138,8 +143,14 @@ async fn stitch( &resolution_string, "-r", "bilinear", + "-co", + "COMPRESS=ZSTD", + "-co", + "PREDICTOR=2", + "-co", + "TILED=YES", "-of", - "BT", + "GTiff", &hgt_index, output.as_str(), ]; @@ -151,33 +162,50 @@ async fn stitch( }) .await?; + tracing::info!("Stitched tile saved to: {output}"); + Ok(output) } -/// Re-purpose the new tile's extent header to instead define its centre. -async fn set_centre_as_extent( - machine: &Arc, - config: &crate::config::Stitch, - file: &str, -) -> Result<()> { - let lon = config.centre.0.to_string(); - let lat = config.centre.1.to_string(); - let arguments = vec![ - "-a_ullr", - lon.as_str(), - lat.as_str(), - lon.as_str(), - lat.as_str(), - file, - ]; +/// Decide on the resolution to use for the stitched tile. +fn get_resolution(config: &crate::config::Stitch) -> f32 { + if let Some(resolution) = config.scale { + return resolution; + } + let vertical_resolution = calculate_longitude_resolution(config.centre.1); + #[expect( + clippy::as_conversions, + clippy::cast_possible_truncation, + reason = "This doesn't need accuracy" + )] + let resolution = (vertical_resolution.midpoint(SRTM3_RESOLUTION_AT_EQUATOR)) as f32; + resolution +} - machine - .command(crate::atlas::machines::connection::Command { - executable: "gdal_edit.py".into(), - args: arguments, - ..Default::default() - }) - .await?; +/// Calculate the vertical height of a pixel in the SRTM3 dataset. Because SRTM is degree-based +/// then the height of a pixel changes depending on how far it is from the equator. +fn calculate_longitude_resolution(latitude_degrees: f64) -> f64 { + const ARC_SECONDS: f64 = 3.0; // For SRTM3 + const SECONDS_PER_DEGREE: f64 = 3600.0; + let earth_radius_meters = f64::from(EARTH_RADIUS * 1000.0); + let latitude_radians = latitude_degrees.to_radians(); + let arc_second_radians = (ARC_SECONDS / SECONDS_PER_DEGREE).to_radians(); - Ok(()) + earth_radius_meters * latitude_radians.cos() * arc_second_radians +} + +#[expect(clippy::float_cmp, reason = "Just tests")] +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn pixel_height_at_latitude() { + assert_eq!( + calculate_longitude_resolution(0.0), + SRTM3_RESOLUTION_AT_EQUATOR + ); + assert_eq!(calculate_longitude_resolution(45.0), 65.595639015131f64); + assert_eq!(calculate_longitude_resolution(-45.0), 65.595639015131f64); + } } diff --git a/crates/tasks/src/tile.rs b/crates/tasks/src/tile.rs index b4606ed..735446a 100644 --- a/crates/tasks/src/tile.rs +++ b/crates/tasks/src/tile.rs @@ -11,7 +11,7 @@ use shared::projector::LonLatCoord; pub struct Tile { /// The centre of the tile. pub centre: shared::projector::LonLatCoord, - /// The width of the tile. Therefore, not the distance from the centre to an edge. Width is + /// The width of the tile in meters. Therefore, not the distance from the centre to an edge. Width is /// better than radius because it defines the minimum line of sight distance we are interested /// in. pub width: f32, @@ -112,7 +112,7 @@ impl Tile { } /// Canonical filename for the tile. - pub fn cog_filename(&self) -> String { + pub fn canonical_filename(&self) -> String { format!("{}_{}.tiff", self.centre.0.x, self.centre.0.y) } } diff --git a/screenshot.webp b/screenshot.webp new file mode 100644 index 0000000..44be347 Binary files /dev/null and b/screenshot.webp differ diff --git a/scripts/cache_tiles.bash b/scripts/cache_tiles.bash index d068ad0..e2a79a1 100644 --- a/scripts/cache_tiles.bash +++ b/scripts/cache_tiles.bash @@ -14,7 +14,7 @@ function cache_cacheable_tiles { export -f cache_tile # Takes about 15 minutes for zoom levels 0-8 - for z in {0..1}; do + for z in {0..8}; do max=$((2 ** z - 1)) parallel --jobs 20 cache_tile "$version $z {1} {2}" ::: $(seq 0 $max) ::: $(seq 0 $max) done @@ -32,7 +32,7 @@ function cache_tile { local x=$3 local y=$4 local path="$z/$x/$y" - local base="runs/$version/pmtiles/world.pmtiles/world" + local base="runs/$version/pmtiles/world" curl \ --create-dirs \ diff --git a/scripts/cloud_init_ubuntu22.bash b/scripts/cloud_init_ubuntu22.bash index bd1d0d3..9eb2a37 100644 --- a/scripts/cloud_init_ubuntu22.bash +++ b/scripts/cloud_init_ubuntu22.bash @@ -31,7 +31,7 @@ function cloud_init_ubuntu22 { ssh "$address" "\ set -euo pipefail source ~/.cargo/env - ./benchmarks/run.sh cpu + cd ~/tvs && ./benchmarks/run.sh cpu cd ~/tvs && RUSTFLAGS='-Ctarget-cpu=native' cargo build --release rm -r ~/.rustup/ rm -r ~/.cargo/ diff --git a/scripts/google_cloud.bash b/scripts/google_cloud.bash index 9f4131a..1503255 100644 --- a/scripts/google_cloud.bash +++ b/scripts/google_cloud.bash @@ -3,7 +3,7 @@ function spin_google_cloud { gcloud compute instances create "$1" --format="json" \ - --zone=us-central1-b \ + --zone=us-central1-a \ --machine-type=h4d-standard-192 \ --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \ --metadata=startup-script="sudo rm -r /usr/lib/google-cloud-sdk/",ssh-keys=atlas:"$2" \ diff --git a/scripts/make_pmtiles.bash b/scripts/make_pmtiles.bash index 147f600..7671c7f 100644 --- a/scripts/make_pmtiles.bash +++ b/scripts/make_pmtiles.bash @@ -9,6 +9,7 @@ function make_pmtiles { # Output `.pmtile` tile. local output=$2 + # Resolution in metres of the output .pmtile local resolution=${3} ensure_tiles_env diff --git a/scripts/prepare_for_cloud.bash b/scripts/prepare_for_cloud.bash deleted file mode 100644 index 186deb3..0000000 --- a/scripts/prepare_for_cloud.bash +++ /dev/null @@ -1,67 +0,0 @@ -function prepare_for_cloud { - set -e - set -x - - # Input `.bt` tile - local input=$1 - # Output `.tiff` tile - local output=$2 - - ensure_tiles_env - - # Lon/lat of the input tile - longitude=$(get_tiff_longitude "$input") - latitude=$(get_tiff_latitude "$input") - # Pixel resolution of the input tile - pixel_width=$(gdalinfo -json "$input" | jq '.size[0]') - # Width of the input tile - width=$((pixel_width * 50)) - - if [ -z "${output:-}" ]; then - output="$longitude"_"$latitude".tiff - echo "⚠️ Using tile's internal lon/lat for filename." - echo "⚠️ This may not match the coordinates of the Packer-defined tile." - fi - - # Where to keep the longest lines COG files. - LONGEST_LINES_DIR=${LONGEST_LINES_DIR:-$OUTPUT_DIR/longest_lines} - - # Just the filename without its path - base=$(basename "$input") - # The filename without its extension - stem="${base%.*}" - - mkdir -p "$LONGEST_LINES_DIR" - mkdir -p "$TMP_DIR" - rm -f "$TMP_DIR/"* - - plain_tif=$TMP_DIR/plain.tif - - # Convert to GeoTiff. - gdal_translate \ - -of GTiff \ - -a_nodata 0 \ - -co COMPRESS=DEFLATE \ - "$input" "$plain_tif" - - # The `.bt` format only has minimal support for georeferencing, so here we edit - # the GeoTiff's projection and extent. This is merely updating header metadata, - # it's not actually interpolating or anything like that. - gdal_edit.py \ - -a_ullr "-$width" "$width" "$width" "-$width" \ - -a_srs "+proj=aeqd +lat_0=$latitude +lon_0=$longitude +datum=WGS84" \ - "$plain_tif" - - if [[ $stem == "longest_lines" ]]; then - # Create the longest line of sight COG. It is used as-is by the website. - cog=$LONGEST_LINES_DIR/"$output" - gdal_translate \ - -of COG \ - -co BLOCKSIZE=32 \ - -co RESAMPLING=NEAREST \ - -co OVERVIEWS=NONE \ - -co COMPRESS=DEFLATE \ - -co PREDICTOR=3 \ - "$plain_tif" "$cog" - fi -} diff --git a/website/public/.assetsignore b/website/public/.assetsignore new file mode 100644 index 0000000..abade04 --- /dev/null +++ b/website/public/.assetsignore @@ -0,0 +1,3 @@ +longest_lines/ +*.pmtiles + diff --git a/website/src/Home.svelte b/website/src/Home.svelte index 51c0406..fd9dd12 100644 --- a/website/src/Home.svelte +++ b/website/src/Home.svelte @@ -124,8 +124,6 @@ }); -
- @@ -163,12 +161,6 @@ diff --git a/website/src/Packing.svelte b/website/src/Packing.svelte index 62c7d99..71cc27d 100644 --- a/website/src/Packing.svelte +++ b/website/src/Packing.svelte @@ -46,14 +46,14 @@ } const tilesCSV = await result.text(); - state.map?.addSource('my-geojson', { + state.map?.addSource('packing', { type: 'geojson', data: tilesCSVToGeoJSON(tilesCSV), }); state.map?.addLayer({ id: 'fills', type: 'fill', - source: 'my-geojson', + source: 'packing', paint: { 'fill-outline-color': '#ff1234', 'fill-color': '#ff0000', @@ -168,6 +168,3 @@ - - diff --git a/website/src/TileWorker.ts b/website/src/TileWorker.ts index 4cdd1cc..90717e8 100644 --- a/website/src/TileWorker.ts +++ b/website/src/TileWorker.ts @@ -41,13 +41,15 @@ self.onmessage = async (event: MessageEvent) => { let bytes: Uint8Array | ArrayBuffer; let url = `${pmtilesSource}/${z}/${x}/${y}.bin${CACHE_BUSTER}`; - if (z <= 8) { + if (pmtilesSource.endsWith('world') && z <= 8) { // For zoom levels 0-8 we skip the Cloudflare Worker and use a CDN cache. This saves // on Cloudflare Worker monthly quotas. url = url .replace(`https://${MAP_SERVER_SUBDOMAIN}.`, 'https://cdn.') - .replace('world.pmtiles/world', 'cache') + .replace('pmtiles/world', 'pmtiles/cache') .replace('.bin', ''); + } else { + console.warn('Not using CDN for tiles at low zoom levels'); } if (isProductionMapServer) { diff --git a/website/src/lib/getLongestLine.ts b/website/src/lib/getLongestLine.ts index 7fb0983..d0d9c87 100644 --- a/website/src/lib/getLongestLine.ts +++ b/website/src/lib/getLongestLine.ts @@ -9,6 +9,7 @@ import { clamp, endLoadingSpinner, getElevationAtCoord, + getScaleFromCog, Log, startLoadingSpinner, VERSION, @@ -245,8 +246,7 @@ async function findNearestCOGURLs(coordinate: LngLat) { const cogFilenames = []; for (const [filename, cog] of cogsIndex) { const distance = coordinate.distanceTo(cog.centre); - const scale = 100; // TODO: I thought we decided to set this in the indexer? - const radius = cog.width / 2 / scale; + const radius = cog.width / 2; Log.debug( `👀 Checking Longest Line COG: ${filename}`, `with centre: ${cog.centre} and radius ${radius}`, @@ -338,8 +338,7 @@ export function convertRasterXYToLngLat( geo.ProjCenterLongGeoKey, geo.ProjCenterLatGeoKey, ); - const resolution = cog.getResolution(); - const scale = Math.abs(resolution[0]); + const scale = getScaleFromCog(cog); const maxIndex = cog.getWidth() - 1.0; const offset = maxIndex / 2.0; const y_flipped = maxIndex - y; diff --git a/website/src/lib/renderLongestLine.ts b/website/src/lib/renderLongestLine.ts index a299f5f..54dfaa0 100644 --- a/website/src/lib/renderLongestLine.ts +++ b/website/src/lib/renderLongestLine.ts @@ -6,7 +6,6 @@ import { state } from '../state.svelte.ts'; import { getLongestLine, makeGoogleEarthLink } from './getLongestLine.ts'; import { - ANGLE_SHIFT, aeqdProjectionString, computeBBox, disablePointer, @@ -71,7 +70,7 @@ export async function render(lngLat: LngLat) { console.log(longestLine); } - longestLine.angle = longestLine.angle + ANGLE_SHIFT; + longestLine.angle = longestLine.angle - 90; const θ = toRadians(longestLine.angle); const dx = longestLine.distance * Math.cos(θ); diff --git a/website/src/lib/utils.ts b/website/src/lib/utils.ts index 0fbdee6..f6e55d0 100644 --- a/website/src/lib/utils.ts +++ b/website/src/lib/utils.ts @@ -1,23 +1,21 @@ import nprogress from 'accessible-nprogress'; +import type { GeoTIFFImage } from 'geotiff'; import { LngLat, LngLatBounds } from 'maplibre-gl'; import type { AppState } from '../state.svelte'; -export const VERSION = 'ryan-fullworld-raw'; +export const VERSION = 'ryan-world2-run-refraction23'; export const CDN_BUCKET = 'https://cdn.alltheviews.world'; export const MAP_SERVER_SUBDOMAIN = 'pmtiles'; export const MAP_SERVER = `https://${MAP_SERVER_SUBDOMAIN}.alltheviews.world`; -export const WORLD_PMTILES = 'world.pmtiles/world'; // TODO move the file to its proper place +export const WORLD_PMTILES = 'world'; export const PMTILES_SERVER = `${MAP_SERVER}/runs/${VERSION}/pmtiles/${WORLD_PMTILES}`; // This is for busting Cloudflare asset cache. Like for an updated `world.pmtiles`, // longest lines index, etc. -export const CACHE_BUSTER = '?buster=23:05-19/04/2026'; +export const CACHE_BUSTER = '?buster=18:36-30/04/2026'; export const EARTH_RADIUS = 6371_000.0; -// Inherited from the TVS algorithm. It's to counter unfavourable floating point rounding. -export const ANGLE_SHIFT = 0.0001; - export const Log = { // biome-ignore lint/suspicious/noExplicitAny: needed for debugging. debug: (...data: any[]) => { @@ -202,3 +200,9 @@ export async function getElevationAtCoord(coord: LngLat) { const payload = await result.json(); return payload.geoPoints[0].elevation; } + +export function getScaleFromCog(cog: GeoTIFFImage) { + const resolution = cog.getResolution(); + const scale = Math.abs(resolution[0]); + return scale; +} diff --git a/website/src/lib/worldLines.ts b/website/src/lib/worldLines.ts index 79b35f7..d15999a 100644 --- a/website/src/lib/worldLines.ts +++ b/website/src/lib/worldLines.ts @@ -18,7 +18,8 @@ import { } from './utils'; /// The filename for the longest lines grid. -const LONGEST_LINES_GRIDED_FILENAME = 'longest_lines_grided.bin'; +const LONGEST_LINES_GRIDED_FILENAME = + 'longest_lines_cogs/longest_lines_grided.bin'; export class LongestLineH3 { coordinate: LngLat;