fix: syncing and provision benchmark

README.md

@@ -120,6 +120,7 @@ RUST_LOG=info,axum=trace,apalis=trace,tasks=trace \
 ```
 
 Add tile jobs:
+(tiles are automatically pulled down from s3://viewview/stitched)
 
 ```
 # Saving data locally, useful for development:
@@ -130,17 +131,15 @@ RUST_LOG=trace cargo run --bin tasks -- atlas run \
   --centre -4.549624919891357,47.44954299926758 \
   --amount 1 \
   --skip 10 \
-  --tvs-executable ../total-viewsheds/target/release/total-viewsheds \
-  --longest-lines-cogs website/public/longest_lines
+  --tvs-executable ../total-viewsheds/target/release/total-viewsheds
 
 # Saving data on remote machines and S3, used for production:
 RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \
   --provider digital-ocean \
   --run-id 0.1 \
   --master website/public/tiles.csv \
   --centre -13.949958801269531,57.94995880126953 \
-  --tvs-executable /root/tvs/target/release/total-viewsheds \
-  --longest-lines-cogs output/longest_lines
+  --tvs-executable /root/tvs/target/release/total-viewsheds
 ```
 
 Atlas doesn't run the following commands, you'll want to manually run them after a

crates/tasks/src/atlas/longest_lines/index.rs

@@ -12,19 +12,12 @@ pub async fn compile() -> Result<()> {
         color_eyre::eyre::bail!("Can't save longest lines when there's no current run config.");
     };
 
+    let index_path = "longest_lines_index.txt";
     let tiles = crate::atlas::db::get_completed_tiles().await?;
-    let default_directory = std::path::Path::new(crate::atlas::tile_job::WORKING_DIRECTORY)
-        .join(crate::atlas::tile_job::LONGEST_LINES_DIRECTORY);
-
-    let index_path = config
-        .clone()
-        .longest_lines_cogs
-        .unwrap_or(default_directory)
-        .join("index.txt");
 
     let mut index = Vec::new();
     for tile in tiles {
-        let filename = tile.cog_filename();
+        let filename = tile.canonical_filename();
         let line = format!("{filename} {}", tile.width);
         index.push(line);
     }
@@ -34,15 +27,15 @@ pub async fn compile() -> Result<()> {
         index.len(),
         index_path
     );
-    std::fs::write(index_path.clone(), index.join("\n"))?;
+    std::fs::write(index_path, index.join("\n"))?;
 
     if !config.is_local_run() {
         let destination = format!(
             "s3://viewview/runs/{}/longest_lines_cogs/index.txt",
             config.run_id
         );
         crate::atlas::machines::local::Machine::connection()
-            .sync_file_to_s3(&index_path.display().to_string(), &destination)
+            .sync_file_to_s3(index_path, &destination)
             .await?;
     }
 

crates/tasks/src/atlas/tile_job.rs

@@ -12,8 +12,8 @@ use tokio::sync::Mutex;
 /// The directory where all our viewview input/output goes.
 pub const WORKING_DIRECTORY: &str = "work";
 
-/// The default directory where all the longest lines COGs live.
-pub const LONGEST_LINES_DIRECTORY: &str = "longest_lines";
+/// Filename for longest lines COG.
+const LONGEST_LINES_COG: &str = "longest_lines.cog.tiff";
 
 #[derive(Debug, serde::Serialize, serde::Deserialize)]
 /// A worker job that processes a tile.
@@ -196,28 +196,61 @@ impl TileRunner<'_> {
 
     /// Process the assets needed to display the output on the website.
     async fn assets(&self) -> Result<()> {
+        self.make_longest_lines_cog().await?;
+
         if !self.job.config.is_local_run() {
+            self.s3_put_longest_lines_cog().await?;
             self.s3_put_raw_tvs_tiff().await?;
         }
 
-        if !self.job.config.is_local_run() {
-            self.s3_put_longest_lines_cog(&self.job.tile.cog_filename())
-                .await?;
-        }
+        Ok(())
+    }
+
+    /// It needs to be a COG because it's queried from the browser.
+    async fn make_longest_lines_cog(&self) -> Result<()> {
+        let plain_tif = format!("{}/longest_lines.tiff", self.job_directory);
+        let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG);
+
+        let args = vec![
+            "-of",
+            "COG",
+            // Smallest valid size (cos we're just querying for single raster points)
+            "-co",
+            "BLOCKSIZE=128",
+            "-co",
+            "RESAMPLING=NEAREST",
+            "-co",
+            "OVERVIEWS=NONE",
+            "-co",
+            "COMPRESS=DEFLATE",
+            "-co",
+            "PREDICTOR=3",
+            &plain_tif,
+            &cog,
+        ];
+
+        self.machine
+            .command(crate::atlas::machines::connection::Command {
+                executable: "/usr/bin/gdal_translate".into(),
+                args,
+                env: vec![],
+                ..Default::default()
+            })
+            .await?;
 
         Ok(())
     }
 
     /// Sync the raw, pre-projected finished heatmap for the tile to our S3 bucket.
     ///
-    /// There isn't a huge difference between this and the post-processed one, but it's a shame
-    /// to have to recompute the entire planet just to get hold of this.
+    /// It's tempting to do the full post-processing here, but it's a lossy step, so any mistake
+    /// and we can't get back the data.
     async fn s3_put_raw_tvs_tiff(&self) -> Result<()> {
-        let tvs_tiff = self.job.tile.cog_filename();
-        let source = format!("{}/tmp/plain.tif", self.job_directory);
+        let source = format!("{}/total_surfaces.tiff", self.job_directory);
         let destination = format!(
-            "s3://viewview/runs/{}/raw/{tvs_tiff}",
-            self.job.config.run_id
+            "s3://viewview/runs/{}/raw/{}",
+            self.job.config.run_id,
+            self.job.tile.canonical_filename()
         );
 
         self.machine.sync_file_to_s3(&source, &destination).await?;
@@ -226,25 +259,16 @@ impl TileRunner<'_> {
     }
 
     /// Sync a longest lines COG to our S3 bucket.
-    async fn s3_put_longest_lines_cog(&self, filename: &str) -> Result<()> {
-        let source_cogs = self
-            .job
-            .config
-            .longest_lines_cogs
-            .clone()
-            .unwrap_or_else(|| LONGEST_LINES_DIRECTORY.into())
-            .join(filename)
-            .display()
-            .to_string();
-
-        let source = format!("{}/{source_cogs}", self.job_directory);
-
+    /// It needs to be a COG because it's queried from the browser.
+    async fn s3_put_longest_lines_cog(&self) -> Result<()> {
+        let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG);
         let destination = format!(
-            "s3://viewview/runs/{}/longest_lines_cogs/{filename}",
-            self.job.config.run_id
+            "s3://viewview/runs/{}/longest_lines_cogs/{}",
+            self.job.config.run_id,
+            self.job.tile.canonical_filename()
         );
 
-        self.machine.sync_file_to_s3(&source, &destination).await?;
+        self.machine.sync_file_to_s3(&cog, &destination).await?;
 
         Ok(())
     }

crates/tasks/src/config.rs

@@ -186,10 +186,6 @@ pub struct Atlas {
     #[arg(long, value_name = "TVS executable")]
     pub tvs_executable: std::path::PathBuf,
 
-    /// Where to load/save longest lines COGs.
-    #[arg(long, value_name = "Longest lines COGs directory")]
-    pub longest_lines_cogs: Option<std::path::PathBuf>,
-
     /// Where to run the computations, locally or on a cloud provider.
     #[arg(
         long,

crates/tasks/src/tile.rs

@@ -112,7 +112,7 @@ impl Tile {
     }
 
     /// Canonical filename for the tile.
-    pub fn cog_filename(&self) -> String {
+    pub fn canonical_filename(&self) -> String {
         format!("{}_{}.tiff", self.centre.0.x, self.centre.0.y)
     }
 }

scripts/cloud_init_ubuntu22.bash

@@ -31,7 +31,7 @@ function cloud_init_ubuntu22 {
 	ssh "$address" "\
 		set -euo pipefail
 	  source ~/.cargo/env
-	  ./benchmarks/run.sh cpu
+	  cd ~/tvs && ./benchmarks/run.sh cpu
 	  cd ~/tvs && RUSTFLAGS='-Ctarget-cpu=native' cargo build --release
 	  rm -r ~/.rustup/
 	  rm -r ~/.cargo/

scripts/google_cloud.bash

@@ -3,7 +3,7 @@
 
 function spin_google_cloud {
   gcloud compute instances create "$1" --format="json" \
-      --zone=us-central1-b \
+      --zone=us-central1-a \
       --machine-type=h4d-standard-192 \
       --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \
       --metadata=startup-script="sudo rm -r /usr/lib/google-cloud-sdk/",ssh-keys=atlas:"$2" \