World run: ryan-world2-run-refraction23

.github/workflows/ci.yaml

@@ -45,6 +45,17 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Install Linux system dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install libgdal-dev gdal-bin
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: 1.94.1
+          components: clippy,rustfmt
       - name: Install pnpm
         uses: pnpm/action-setup@v4
         with:
@@ -60,3 +71,8 @@ jobs:
         working-directory: website
         run: node_modules/.bin/biome check
 
+      - name: Lint
+        run: cargo clippy -- --deny warnings
+      - name: Formatting
+        run: cargo fmt --check
+

README.md

@@ -1,5 +1,7 @@
 # A View Of All Views
 
+![Screenshot of app](screenshot.webp)
+
 This repo is for all the supporting code used to find and display the longest line of sight on the planet.
 
 The main viewshed algorithm is another repo https://github.com/AllTheLines/CacheTVS
@@ -69,26 +71,33 @@ So can we strike an optimal balance? This is what the `Packer` in this repo trie
 
 Creates arbitrary tiles out of the global DEM data.
 
+Stitch one:
 ```
 cargo run --bin tasks -- stitch \
   --dems /publicish/dems \
   --centre -3.049208402633667,53.24937438964844 \
   --width 366610.1875
 ```
+(optionally takes `--scale` argument)
+
+Stitch all from a folder:
+```
+cargo run --bin tasks -- atlas stitch-all \
+  --tmp-directory /mnt/disks/data/tmp \
+  --dems /mnt/disks/data/srtm/ \
+  --master website/public/tiles.csv
+```
+(optionally takes `--scale` argument)
 
 ## Calculate Total Viewsheds
 
 Using https://github.com/AllTheLines/CacheTVS
 
-Outputs `.bt` heatmap.
+Outputs `total_surfaces.tiff` and `longest_lines.tiff`.
 
 ## Prepare For Cloud
 
 ```
-# Note that all these depend on a `./output` path existing.
-./ctl.sh prepare_for_cloud ../total-viewsheds/output/total_surfaces.bt
-./ctl.sh prepare_for_cloud ../total-viewsheds/output/longest_lines.bt
-
 ./ctl.sh make_pmtiles latest website/public/world.pmtiles
 ```
 
@@ -113,6 +122,7 @@ RUST_LOG=info,axum=trace,apalis=trace,tasks=trace \
 ```
 
 Add tile jobs:
+(tiles are automatically pulled down from s3://viewview/stitched)
 
 ```
 # Saving data locally, useful for development:
@@ -123,30 +133,30 @@ RUST_LOG=trace cargo run --bin tasks -- atlas run \
   --centre -4.549624919891357,47.44954299926758 \
   --amount 1 \
   --skip 10 \
-  --tvs-executable ../total-viewsheds/target/release/total-viewsheds \
-  --longest-lines-cogs website/public/longest_lines
+  --tvs-executable ../total-viewsheds/target/release/total-viewsheds
 
 # Saving data on remote machines and S3, used for production:
 RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \
   --provider digital-ocean \
   --run-id 0.1 \
   --master website/public/tiles.csv \
   --centre -13.949958801269531,57.94995880126953 \
-  --tvs-executable /root/tvs/target/release/total-viewsheds \
-  --longest-lines-cogs output/longest_lines
+  --tvs-executable /root/tvs/target/release/total-viewsheds
 ```
 
 Atlas doesn't run the following commands, you'll want to manually run them after a
 bunch of tiles have been processed:
 
-* Create the gigantic (10s of GBs) global `.pmtile` that contains the TVS heatmap for the entire planet.
+* Create the gigantic (~500GB) global `.pmtile` that contains the TVS heatmap for the entire planet.
 ```
 # This requires a machine with a lot of disk, RAM and CPU. As of writing, I'd recommend
 # 3TB disk, >150GB RAM and at least 48 cores.
 # Replace `latest` with `local` to skip syncing files to S3.
-./ctl.sh make_pmtiles latest work/world.pmtiles
+./ctl.sh make_pmtiles latest work/world.pmtiles 100
 ```
 
+Note that this process creates a _temporary_ version of the `.pmtile`, and then copies that to its final destination. So you need at least 1.2TB of disk for a world PMTile. And then of course you need space to download the raw .tiffs, and space to save the post-processed versions of those .tiffs. So at least 2TB is a good idea.
+
 * Create an index of all the COG (optimised GeoTiff) files that contain all the longest lines of sight for every point on the planet. Should only take seconds to run.
 ```
 RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas longest-lines-index
@@ -158,7 +168,7 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas longest-lines-index
 
 RUST_LOG=off,tasks=trace cargo run --release --bin tasks -- \
   atlas longest-lines-overviews \
-  --tiffs work/longest_lines
+  --tiffs work/longest_lines --run-id 0.1
 ```
 
 ## Static Site Website

crates/shared/src/lib.rs

@@ -8,4 +8,3 @@
 pub mod metadata;
 pub mod projector;
 pub mod utils;
-

crates/shared/src/projector.rs

@@ -3,7 +3,7 @@
 use color_eyre::{Result, eyre::ContextCompat as _};
 
 /// The radius of the planet in kilometers.
-pub const EARTH_RADIUS: f32 = 6371.0;
+pub const EARTH_RADIUS: f32 = 6378.137;
 
 /// A longtitude/latitude coordinate.
 #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Default)]

crates/tasks/src/atlas/daemon.rs

@@ -7,8 +7,7 @@ use apalis::{layers::WorkerBuilderExt as _, prelude::Layer as _};
 /// The URL for the worker web UI.
 const WEB_UI_HOST: &str = "localhost:3003";
 
-
-/// Start the Atlas daemon
+/// Start the Atlas daemon.
 pub async fn start_all(
     _: &crate::config::Worker,
     broadcaster: std::sync::Arc<std::sync::Mutex<apalis_board_api::sse::TracingBroadcaster>>,
@@ -53,10 +52,8 @@ async fn start_existing_machines() -> Result<()> {
     );
 
     for job in jobs {
-        crate::atlas::machines::new_machine_job::new_machine_handler(
-            job.machine.as_ref().clone(),
-        )
-        .await?;
+        crate::atlas::machines::new_machine_job::new_machine_handler(job.machine.as_ref().clone())
+            .await?;
     }
 
     Ok(())

crates/tasks/src/atlas/longest_lines/grided.rs

@@ -8,9 +8,9 @@ use tokio::io::AsyncWriteExt as _;
 #[repr(C)]
 #[derive(Copy, Clone, Debug, bytemuck::Zeroable, bytemuck::Pod)]
 pub struct Grided {
-    /// Longtitude
+    /// Longtitude.
     pub lon: f32,
-    /// Latitude
+    /// Latitude.
     pub lat: f32,
     /// The distance of the line of sight in meters.
     pub distance: u32,

crates/tasks/src/atlas/longest_lines/index.rs

@@ -12,23 +12,13 @@ pub async fn compile() -> Result<()> {
         color_eyre::eyre::bail!("Can't save longest lines when there's no current run config.");
     };
 
+    let index_path = "longest_lines_index.txt";
     let tiles = crate::atlas::db::get_completed_tiles().await?;
-    let default_directory = std::path::Path::new(crate::atlas::tile_job::WORKING_DIRECTORY)
-        .join(crate::atlas::tile_job::LONGEST_LINES_DIRECTORY);
-
-    let index_path = config
-        .clone()
-        .longest_lines_cogs
-        .unwrap_or(default_directory)
-        .join("index.txt");
 
     let mut index = Vec::new();
     for tile in tiles {
-        let filename = tile.cog_filename();
-        let line = format!(
-            "{filename} {}",
-            tile.width * crate::atlas::tile_job::DEM_SCALE
-        );
+        let filename = tile.canonical_filename();
+        let line = format!("{filename} {}", tile.width);
         index.push(line);
     }
 
@@ -37,15 +27,15 @@ pub async fn compile() -> Result<()> {
         index.len(),
         index_path
     );
-    std::fs::write(index_path.clone(), index.join("\n"))?;
+    std::fs::write(index_path, index.join("\n"))?;
 
     if !config.is_local_run() {
         let destination = format!(
             "s3://viewview/runs/{}/longest_lines_cogs/index.txt",
             config.run_id
         );
         crate::atlas::machines::local::Machine::connection()
-            .sync_file_to_s3(&index_path.display().to_string(), &destination)
+            .sync_file_to_s3(index_path, &destination)
             .await?;
     }