diff --git a/README.md b/README.md index ddc6355..b2a4a4e 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ RUST_LOG=info,axum=trace,apalis=trace,tasks=trace \ ``` Add tile jobs: +(tiles are automatically pulled down from s3://viewview/stitched) ``` # Saving data locally, useful for development: @@ -130,8 +131,7 @@ RUST_LOG=trace cargo run --bin tasks -- atlas run \ --centre -4.549624919891357,47.44954299926758 \ --amount 1 \ --skip 10 \ - --tvs-executable ../total-viewsheds/target/release/total-viewsheds \ - --longest-lines-cogs website/public/longest_lines + --tvs-executable ../total-viewsheds/target/release/total-viewsheds # Saving data on remote machines and S3, used for production: RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ @@ -139,8 +139,7 @@ RUST_LOG=off,tasks=trace cargo run --bin tasks -- atlas run \ --run-id 0.1 \ --master website/public/tiles.csv \ --centre -13.949958801269531,57.94995880126953 \ - --tvs-executable /root/tvs/target/release/total-viewsheds \ - --longest-lines-cogs output/longest_lines + --tvs-executable /root/tvs/target/release/total-viewsheds ``` Atlas doesn't run the following commands, you'll want to manually run them after a diff --git a/crates/tasks/src/atlas/longest_lines/index.rs b/crates/tasks/src/atlas/longest_lines/index.rs index 115e31b..5541ca3 100644 --- a/crates/tasks/src/atlas/longest_lines/index.rs +++ b/crates/tasks/src/atlas/longest_lines/index.rs @@ -12,19 +12,12 @@ pub async fn compile() -> Result<()> { color_eyre::eyre::bail!("Can't save longest lines when there's no current run config."); }; + let index_path = "longest_lines_index.txt"; let tiles = crate::atlas::db::get_completed_tiles().await?; - let default_directory = std::path::Path::new(crate::atlas::tile_job::WORKING_DIRECTORY) - .join(crate::atlas::tile_job::LONGEST_LINES_DIRECTORY); - - let index_path = config - .clone() - .longest_lines_cogs - .unwrap_or(default_directory) - .join("index.txt"); let mut index = Vec::new(); for tile in tiles { - let filename = tile.cog_filename(); + let filename = tile.canonical_filename(); let line = format!("{filename} {}", tile.width); index.push(line); } @@ -34,7 +27,7 @@ pub async fn compile() -> Result<()> { index.len(), index_path ); - std::fs::write(index_path.clone(), index.join("\n"))?; + std::fs::write(index_path, index.join("\n"))?; if !config.is_local_run() { let destination = format!( @@ -42,7 +35,7 @@ pub async fn compile() -> Result<()> { config.run_id ); crate::atlas::machines::local::Machine::connection() - .sync_file_to_s3(&index_path.display().to_string(), &destination) + .sync_file_to_s3(index_path, &destination) .await?; } diff --git a/crates/tasks/src/atlas/tile_job.rs b/crates/tasks/src/atlas/tile_job.rs index 8368aed..3c84d86 100644 --- a/crates/tasks/src/atlas/tile_job.rs +++ b/crates/tasks/src/atlas/tile_job.rs @@ -12,8 +12,8 @@ use tokio::sync::Mutex; /// The directory where all our viewview input/output goes. pub const WORKING_DIRECTORY: &str = "work"; -/// The default directory where all the longest lines COGs live. -pub const LONGEST_LINES_DIRECTORY: &str = "longest_lines"; +/// Filename for longest lines COG. +const LONGEST_LINES_COG: &str = "longest_lines.cog.tiff"; #[derive(Debug, serde::Serialize, serde::Deserialize)] /// A worker job that processes a tile. @@ -196,28 +196,61 @@ impl TileRunner<'_> { /// Process the assets needed to display the output on the website. async fn assets(&self) -> Result<()> { + self.make_longest_lines_cog().await?; + if !self.job.config.is_local_run() { + self.s3_put_longest_lines_cog().await?; self.s3_put_raw_tvs_tiff().await?; } - if !self.job.config.is_local_run() { - self.s3_put_longest_lines_cog(&self.job.tile.cog_filename()) - .await?; - } + Ok(()) + } + + /// It needs to be a COG because it's queried from the browser. + async fn make_longest_lines_cog(&self) -> Result<()> { + let plain_tif = format!("{}/longest_lines.tiff", self.job_directory); + let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG); + + let args = vec![ + "-of", + "COG", + // Smallest valid size (cos we're just querying for single raster points) + "-co", + "BLOCKSIZE=128", + "-co", + "RESAMPLING=NEAREST", + "-co", + "OVERVIEWS=NONE", + "-co", + "COMPRESS=DEFLATE", + "-co", + "PREDICTOR=3", + &plain_tif, + &cog, + ]; + + self.machine + .command(crate::atlas::machines::connection::Command { + executable: "/usr/bin/gdal_translate".into(), + args, + env: vec![], + ..Default::default() + }) + .await?; Ok(()) } /// Sync the raw, pre-projected finished heatmap for the tile to our S3 bucket. /// - /// There isn't a huge difference between this and the post-processed one, but it's a shame - /// to have to recompute the entire planet just to get hold of this. + /// It's tempting to do the full post-processing here, but it's a lossy step, so any mistake + /// and we can't get back the data. async fn s3_put_raw_tvs_tiff(&self) -> Result<()> { - let tvs_tiff = self.job.tile.cog_filename(); - let source = format!("{}/tmp/plain.tif", self.job_directory); + let source = format!("{}/total_surfaces.tiff", self.job_directory); let destination = format!( - "s3://viewview/runs/{}/raw/{tvs_tiff}", - self.job.config.run_id + "s3://viewview/runs/{}/raw/{}", + self.job.config.run_id, + self.job.tile.canonical_filename() ); self.machine.sync_file_to_s3(&source, &destination).await?; @@ -226,25 +259,16 @@ impl TileRunner<'_> { } /// Sync a longest lines COG to our S3 bucket. - async fn s3_put_longest_lines_cog(&self, filename: &str) -> Result<()> { - let source_cogs = self - .job - .config - .longest_lines_cogs - .clone() - .unwrap_or_else(|| LONGEST_LINES_DIRECTORY.into()) - .join(filename) - .display() - .to_string(); - - let source = format!("{}/{source_cogs}", self.job_directory); - + /// It needs to be a COG because it's queried from the browser. + async fn s3_put_longest_lines_cog(&self) -> Result<()> { + let cog = format!("{}/{}", self.job_directory, LONGEST_LINES_COG); let destination = format!( - "s3://viewview/runs/{}/longest_lines_cogs/{filename}", - self.job.config.run_id + "s3://viewview/runs/{}/longest_lines_cogs/{}", + self.job.config.run_id, + self.job.tile.canonical_filename() ); - self.machine.sync_file_to_s3(&source, &destination).await?; + self.machine.sync_file_to_s3(&cog, &destination).await?; Ok(()) } diff --git a/crates/tasks/src/config.rs b/crates/tasks/src/config.rs index 8e3e4dd..53fe632 100644 --- a/crates/tasks/src/config.rs +++ b/crates/tasks/src/config.rs @@ -186,10 +186,6 @@ pub struct Atlas { #[arg(long, value_name = "TVS executable")] pub tvs_executable: std::path::PathBuf, - /// Where to load/save longest lines COGs. - #[arg(long, value_name = "Longest lines COGs directory")] - pub longest_lines_cogs: Option, - /// Where to run the computations, locally or on a cloud provider. #[arg( long, diff --git a/crates/tasks/src/tile.rs b/crates/tasks/src/tile.rs index 9dd9998..735446a 100644 --- a/crates/tasks/src/tile.rs +++ b/crates/tasks/src/tile.rs @@ -112,7 +112,7 @@ impl Tile { } /// Canonical filename for the tile. - pub fn cog_filename(&self) -> String { + pub fn canonical_filename(&self) -> String { format!("{}_{}.tiff", self.centre.0.x, self.centre.0.y) } } diff --git a/scripts/cloud_init_ubuntu22.bash b/scripts/cloud_init_ubuntu22.bash index bd1d0d3..9eb2a37 100644 --- a/scripts/cloud_init_ubuntu22.bash +++ b/scripts/cloud_init_ubuntu22.bash @@ -31,7 +31,7 @@ function cloud_init_ubuntu22 { ssh "$address" "\ set -euo pipefail source ~/.cargo/env - ./benchmarks/run.sh cpu + cd ~/tvs && ./benchmarks/run.sh cpu cd ~/tvs && RUSTFLAGS='-Ctarget-cpu=native' cargo build --release rm -r ~/.rustup/ rm -r ~/.cargo/ diff --git a/scripts/google_cloud.bash b/scripts/google_cloud.bash index 9f4131a..1503255 100644 --- a/scripts/google_cloud.bash +++ b/scripts/google_cloud.bash @@ -3,7 +3,7 @@ function spin_google_cloud { gcloud compute instances create "$1" --format="json" \ - --zone=us-central1-b \ + --zone=us-central1-a \ --machine-type=h4d-standard-192 \ --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \ --metadata=startup-script="sudo rm -r /usr/lib/google-cloud-sdk/",ssh-keys=atlas:"$2" \