From 58c9e345acd2062199da756821a33a359b2b2786 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Tue, 28 Apr 2026 23:27:59 -0700 Subject: [PATCH 01/44] Configure glm_aed_flare_rs for FaaSr: +use_faasr, +vera4cast_* anonymous datastores --- .../glm_aed_flare_rs/configure_flare_glm_aed.yml | 8 ++++++++ configuration/glm_aed_flare_rs/configure_run.yml | 1 + 2 files changed, 9 insertions(+) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index f35c6a4..ebfb12a 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -20,6 +20,14 @@ s3: scores: endpoint: amnh1.osn.mghpcc.org bucket: bio230121-bucket01/flare/scores/parquet + vera4cast_targets: + endpoint: amnh1.osn.mghpcc.org + bucket: bio230121-bucket01/vera4cast/targets + anonymous: true + vera4cast_forecasts: + endpoint: amnh1.osn.mghpcc.org + bucket: bio230121-bucket01/vera4cast/forecasts/archive-parquet + anonymous: true location: site_id: fcre name: Falling Creek Reservoir diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 34ea50e..60484de 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -7,3 +7,4 @@ sim_name: glm_aed_flare_rs configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml use_s3: TRUE +use_faasr: TRUE From 47ec5296164f717e57917210557c26bdefc6796e Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Tue, 28 Apr 2026 23:31:16 -0700 Subject: [PATCH 02/44] Refactor FCRE helpers: backward-compatible config arg, dispatch via flare_* --- R/convert_vera4cast_inflow.R | 13 +++++++++---- R/generate_forecast_score_arrow.R | 9 +++++++-- .../glm_aed_flare_rs/generate_inflow_forecast.R | 15 +++++++++++++-- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/R/convert_vera4cast_inflow.R b/R/convert_vera4cast_inflow.R index 88ed6a4..e31dc00 100644 --- a/R/convert_vera4cast_inflow.R +++ b/R/convert_vera4cast_inflow.R @@ -1,4 +1,4 @@ -convert_vera4cast_inflow <- function(reference_date, model_id, save_path){ +convert_vera4cast_inflow <- function(reference_date, model_id, save_path, config = NULL){ variables <- c("TP_ugL_sample", "NH4_ugL_sample","NO3NO2_ugL_sample", "SRP_ugL_sample","DOC_mgL_sample","DRSI_mgL_sample", @@ -9,9 +9,14 @@ forecast_df <- NULL for(i in 1:length(variables)){ - s3 <- arrow::s3_bucket(bucket = glue::glue("bio230121-bucket01/vera4cast/forecasts/archive-parquet/project_id=vera4cast/duration=P1D/variable={variables[i]}/model_id={model_id}/reference_date={reference_date}"), - endpoint_override = "https://amnh1.osn.mghpcc.org", - anonymous = TRUE) + if(!is.null(config)){ + faasr_prefix <- glue::glue("project_id=vera4cast/duration=P1D/variable={variables[i]}/model_id={model_id}/reference_date={reference_date}") + s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "vera4cast_forecasts", faasr_prefix = faasr_prefix, config = config) + } else { + s3 <- arrow::s3_bucket(bucket = glue::glue("bio230121-bucket01/vera4cast/forecasts/archive-parquet/project_id=vera4cast/duration=P1D/variable={variables[i]}/model_id={model_id}/reference_date={reference_date}"), + endpoint_override = "https://amnh1.osn.mghpcc.org", + anonymous = TRUE) + } ## test to see if inflow forecast exists ## tryCatch({ diff --git a/R/generate_forecast_score_arrow.R b/R/generate_forecast_score_arrow.R index f51cd38..a0d9fc9 100644 --- a/R/generate_forecast_score_arrow.R +++ b/R/generate_forecast_score_arrow.R @@ -12,10 +12,15 @@ generate_forecast_score_arrow <- function(targets_df, bucket = NULL, endpoint = NULL, local_directory = NULL, - variable_types = "state"){ + variable_types = "state", + config = NULL){ - if(use_s3){ + if(!is.null(config)){ + vars <- arrow_env_vars() + output_directory <- FLAREr::flare_arrow_s3_bucket(server_name = "scores", config = config) + on.exit(unset_arrow_vars(vars)) + }else if(use_s3){ if(is.null(bucket) | is.null(endpoint)){ stop("scoring function needs bucket and endpoint if use_s3=TRUE") } diff --git a/workflows/glm_aed_flare_rs/generate_inflow_forecast.R b/workflows/glm_aed_flare_rs/generate_inflow_forecast.R index e853ea2..706dd6e 100644 --- a/workflows/glm_aed_flare_rs/generate_inflow_forecast.R +++ b/workflows/glm_aed_flare_rs/generate_inflow_forecast.R @@ -7,8 +7,19 @@ config <- FLAREr::set_up_simulation(configure_run_file,lake_directory, config_se print('read VERA targets...') -targets_vera <- readr::read_csv("https://amnh1.osn.mghpcc.org/bio230121-bucket01/vera4cast/targets/project_id=vera4cast/duration=P1D/daily-inflow-targets.csv.gz", - show_col_types = FALSE) +if(!is.null(config$s3$vera4cast_targets)){ + targets_local <- file.path(tempdir(), "daily-inflow-targets.csv.gz") + FLAREr::flare_get_file(local_file = "daily-inflow-targets.csv.gz", + remote_file = "daily-inflow-targets.csv.gz", + server_name = "vera4cast_targets", + local_folder = tempdir(), + remote_folder = "project_id=vera4cast/duration=P1D", + config = config) + targets_vera <- readr::read_csv(targets_local, show_col_types = FALSE) +} else { + targets_vera <- readr::read_csv("https://amnh1.osn.mghpcc.org/bio230121-bucket01/vera4cast/targets/project_id=vera4cast/duration=P1D/daily-inflow-targets.csv.gz", + show_col_types = FALSE) +} inflow_hist_dates <- tibble(datetime = seq(min(targets_vera$datetime), max(targets_vera$datetime), by = "1 day")) From 4840980c2fc56f2d97cd7b5019a72b6777cbaeba Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Tue, 28 Apr 2026 23:34:48 -0700 Subject: [PATCH 03/44] Add run_fcre_aed_forecast: FaaSr-action entry point (preserves combined_run_aed.R) --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R new file mode 100644 index 0000000..1a5eee9 --- /dev/null +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -0,0 +1,290 @@ +run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", + configure_run_file = "configure_run.yml") { + + library(tidyverse) + library(lubridate) + set.seed(100) + + Sys.setenv('GLM_PATH' = 'GLM3r') + options(future.globals.maxSize = 891289600) + + Sys.setenv("AWS_DEFAULT_REGION" = "amnh1", + "AWS_S3_ENDPOINT" = "osn.mghpcc.org", + "USE_HTTPS" = TRUE) + + lake_directory <- here::here() + + source(file.path(lake_directory, "R/convert_vera4cast_inflow.R")) + source(file.path(lake_directory, "R/generate_forecast_score_arrow.R")) + + config <- FLAREr::set_up_simulation(configure_run_file = configure_run_file, + lake_directory = lake_directory, + config_set_name = config_set_name) + + noaa_ready <- FLAREr::check_noaa_present(lake_directory, + configure_run_file, + config_set_name = config_set_name) + + reference_date <- lubridate::as_date(config$run_config$forecast_start_datetime) + inflow_prefix <- "archive-parquet/project_id=vera4cast/duration=P1D/variable=Temp_C_mean/model_id=inflow_gefsClimAED" + s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "vera4cast_forecasts", + faasr_prefix = inflow_prefix, + config = config) + avail_dates <- gsub("reference_date=", "", s3$ls()) + inflow_ready <- reference_date %in% lubridate::as_date(avail_dates) + + message(paste0("noaa ready: ", noaa_ready)) + message(paste0("inflow ready: ", inflow_ready)) + + loop_start <- Sys.time() + loop_budget_seconds <- 5.5 * 60 * 60 + + while (noaa_ready & inflow_ready) { + + if (as.numeric(Sys.time() - loop_start, units = "secs") > loop_budget_seconds) { + message("Approaching action time budget; exiting loop cleanly with restart written.") + break + } + + source(file.path(lake_directory, "workflows", config_set_name, "generate_inflow_forecast.R")) + + insitu_local <- file.path(tempdir(), "daily-insitu-targets.csv.gz") + FLAREr::flare_get_file(local_file = "daily-insitu-targets.csv.gz", + remote_file = "daily-insitu-targets.csv.gz", + server_name = "vera4cast_targets", + local_folder = tempdir(), + remote_folder = "project_id=vera4cast/duration=P1D", + config = config) + + readr::read_csv(insitu_local, show_col_types = FALSE) |> + dplyr::mutate(observation = ifelse(variable == "DO_mgL_mean", observation*1000*(1/32), observation), + observation = ifelse(variable == "fDOM_QSU_mean", -151.3407 + observation*29.62654, observation), + depth_m = ifelse(depth_m == 0.1, 0.0, depth_m)) |> + dplyr::rename(depth = depth_m) |> + dplyr::filter(site_id == "fcre", + datetime >= as_datetime(config$run_config$start_datetime)) |> + dplyr::mutate(datetime = lubridate::as_datetime(datetime)) |> + readr::write_csv(file.path(config$file_path$qaqc_data_directory, + paste0(config$location$site_id, "-targets-insitu.csv"))) + + source(file.path(lake_directory, "workflows", config_set_name, "getLST.R")) + data <- get_lst(bbox, config$run_config$start_datetime, config$run_config$forecast_start_datetime) + vals <- get_vals(points, data) + if (exists("vals") == TRUE) { + clean_data(vals) |> + readr::write_csv(file.path(config$file_path$qaqc_data_directory, + paste0(config$location$site_id, "-targets-rs.csv"))) + } else { + message("No new RS data") + data <- as.data.frame(cbind("datetime" = paste0(substr(as.character(config$run_config$start_datetime), 0, 10), "T00:00:00Z"), + "observation" = NA, + "site_id" = "fcre", + "depth" = 0, + "variable" = "temperature")) + data |> + readr::write_csv(file.path(config$file_path$qaqc_data_directory, + paste0(config$location$site_id, "-targets-rs.csv"))) + } + + FLAREr::run_flare(lake_directory = lake_directory, + configure_run_file = configure_run_file, + config_set_name = config_set_name) + + forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", + config = config) + + ref_date <- as.character(lubridate::as_date(config$run_config$forecast_start_datetime)) + forecast_df <- arrow::open_dataset(forecasts_s3) |> + filter(model_id == "glm_aed_flare_v3", + site_id == "fcre", + reference_date == ref_date) |> + collect() |> + mutate(datetime = lubridate::as_datetime(datetime)) + + vera_variables <- c("Temp_C_mean","Chla_ugL_mean", "DO_mgL_mean", "fDOM_QSU_mean", "NH4_ugL_sample", + "NO3NO2_ugL_sample", "SRP_ugL_sample", "DIC_mgL_sample","Secchi_m_sample", + "Bloom_binary_mean","CH4_umolL_sample","IceCover_binary_max", "CO2flux_umolm2s_mean", "CH4flux_umolm2s_mean", + "Mixed_binary_mean") + + bloom_binary <- forecast_df |> + dplyr::filter(depth == 1.6 & variable == "Chla_ugL_mean") |> + dplyr::mutate(over = ifelse(prediction > 20, 1, 0)) |> + dplyr::summarize(prediction = sum(over) / n(), .by = c(datetime, reference_datetime, model_id, site_id, depth, variable)) |> + dplyr::mutate(family = "bernoulli", + parameter = "prob", + variable = "Bloom_binary_mean", + datetime = lubridate::as_datetime(datetime)) |> + dplyr::rename(depth_m = depth) |> + dplyr::select(reference_datetime, datetime, model_id, site_id, depth_m, family, parameter, variable, prediction) + + ice_binary <- forecast_df |> + dplyr::filter(variable == "ice_thickness") |> + dplyr::mutate(over = ifelse(prediction > 0, 1, 0)) |> + dplyr::summarize(prediction = sum(over) / n(), .by = c(datetime, reference_datetime, model_id, site_id, depth, variable)) |> + dplyr::mutate(family = "bernoulli", + parameter = "prob", + variable = "IceCover_binary_max", + depth = NA, + datetime = lubridate::as_datetime(datetime)) |> + dplyr::rename(depth_m = depth) |> + dplyr::select(reference_datetime, datetime, model_id, site_id, depth_m, family, parameter, variable, prediction) + + min_depth <- 1 + max_depth <- 8 + threshold <- 0.1 + + temp_forecast <- forecast_df |> + filter(variable %in% c("temp_1.0m_mean","temp_8.0m_mean")) |> + mutate(depth = ifelse(variable == "temp_1.0m_mean", 1.0, 8.0), + variable = "Temp_C_mean", + datetime = lubridate::as_datetime(datetime - lubridate::days(1))) |> + pivot_wider(names_from = depth, names_prefix = "wtr_", values_from = prediction) + + colnames(temp_forecast)[which(colnames(temp_forecast) == paste0("wtr_", min_depth))] <- "min_depth" + colnames(temp_forecast)[which(colnames(temp_forecast) == paste0("wtr_", max_depth))] <- "max_depth" + + mix_binary <- temp_forecast |> + mutate(min_depth = rLakeAnalyzer::water.density(min_depth), + max_depth = rLakeAnalyzer::water.density(max_depth), + mixed = ifelse((max_depth - min_depth) < threshold, 1, 0)) |> + summarise(prediction = (sum(mixed)/n()), .by = c(datetime, reference_datetime, model_id, site_id, variable)) |> + dplyr::mutate(family = "bernoulli", + parameter = "prob", + variable = "Mixed_binary_mean", + depth = NA, + datetime = lubridate::as_datetime(datetime)) |> + dplyr::rename(depth_m = depth) |> + dplyr::select(reference_datetime, datetime, model_id, site_id, depth_m, family, parameter, variable, prediction) + + vera4cast_df <- forecast_df |> + dplyr::rename(depth_m = depth) |> + dplyr::mutate(variable = ifelse(variable == "oxy_mean", "DO_mgL_mean", variable), + depth_m = ifelse(variable == "DO_mgL_mean", 1.6, depth_m), + datetime = ifelse(variable == "DO_mgL_mean", datetime - lubridate::days(1), datetime), + prediction = ifelse(variable == "DO_mgL_mean", prediction/1000*(32), prediction), + variable = ifelse(variable == "Temp_C_mean", "Temp_C_mean_all_depth", variable), + variable = ifelse(variable == "temp_1.6m_mean", "Temp_C_mean", variable), + depth_m = ifelse(variable == "Temp_C_mean", 1.6, depth_m), + datetime = ifelse(variable == "Temp_C_mean", datetime - lubridate::days(1), datetime), + prediction = ifelse(variable == "fDOM_QSU_mean", (151.3407 + prediction)/29.62654, prediction), + prediction = ifelse(variable == "NIT_amm", prediction/1000/0.001/(1/18.04), prediction), + variable = ifelse(variable == "NIT_amm", "NH4_ugL_sample", variable), + prediction = ifelse(variable == "NIT_nit", prediction/1000/0.001/(1/62.00), prediction), + variable = ifelse(variable == "NIT_amm", "NO3NO2_ugL_sample", variable), + prediction = ifelse(variable == "PHS_frp", prediction/1000/0.001/(1/94.9714), prediction), + variable = ifelse(variable == "PHS_frp", "SRP_ugL_sample", variable), + prediction = ifelse(variable == "CAR_dic", prediction/1000/(1/52.515), prediction), + variable = ifelse(variable == "CAR_dic", "DIC_mgL_sample", variable), + variable = ifelse(variable == "CAR_ch4", "CH4_umolL_sample", variable), + variable = ifelse(variable == "secchi", "Secchi_m_sample", variable), + prediction = ifelse(variable == "co2_flux_mean", prediction/0.001/86400, prediction), + variable = ifelse(variable == "co2_flux_mean", "CO2flux_umolm2s_mean", variable), + prediction = ifelse(variable == "ch4_flux_mean", prediction/0.001/86400, prediction), + variable = ifelse(variable == "ch4_flux_mean", "CH4flux_umolm2s_mean", variable), + depth_m = ifelse(depth_m == 0.0, 0.1, depth_m), + datetime = lubridate::as_datetime(datetime)) |> + dplyr::select(-forecast, -variable_type) |> + dplyr::mutate(parameter = as.character(parameter)) |> + dplyr::bind_rows(bloom_binary) |> + dplyr::bind_rows(ice_binary) |> + dplyr::bind_rows(mix_binary) |> + dplyr::filter(variable %in% vera_variables) |> + mutate(project_id = "vera4cast", + model_id = config$run_config$sim_name, + family = "ensemble", + site_id = "fcre", + duration = "P1D", + datetime = lubridate::as_datetime(datetime), + reference_datetime = lubridate::as_datetime(reference_datetime)) |> + filter(datetime >= reference_datetime) |> + distinct(reference_datetime, datetime, variable, depth_m, parameter, model_id, .keep_all = TRUE) + + file_name <- paste0(config$run_config$sim_name, "-", + lubridate::as_date(vera4cast_df$reference_datetime[1]), ".csv.gz") + readr::write_csv(vera4cast_df, file = file_name) + + message("Scoring forecasts") + + forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", + config = config) + forecast_df <- arrow::open_dataset(forecasts_s3) |> + dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> + dplyr::filter(model_id == "glm_aed_flare_v3", + site_id == config$location$site_id, + reference_date == lubridate::as_datetime(config$run_config$forecast_start_datetime)) |> + dplyr::collect() + + if (config$output_settings$evaluate_past & config$run_config$use_s3) { + past_days <- lubridate::as_date(lubridate::as_date(config$run_config$forecast_start_datetime) - + lubridate::days(config$run_config$forecast_horizon)) + past_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", + config = config) + past_forecasts <- arrow::open_dataset(past_s3) |> + dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> + dplyr::filter(model_id == "glm_aed_flare_v3", + site_id == config$location$site_id, + reference_date == past_days) |> + dplyr::collect() + } else { + past_forecasts <- NULL + } + + combined_forecasts <- dplyr::bind_rows(forecast_df, past_forecasts) + + targets_df <- read_csv(file.path(config$file_path$qaqc_data_directory, + paste0(config$location$site_id, "-targets-insitu.csv")), + show_col_types = FALSE) + + scoring <- generate_forecast_score_arrow(targets_df = targets_df, + forecast_df = combined_forecasts, + use_s3 = config$run_config$use_s3, + bucket = config$s3$scores$bucket, + endpoint = config$s3$scores$endpoint, + local_directory = "./scores/fcre", + variable_types = c("state", "parameter"), + config = config) + + forecast_start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) + lubridate::days(1) + start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) - lubridate::days(4) + restart_file <- paste0(config$location$site_id, "-", + (lubridate::as_date(forecast_start_datetime) - days(1)), + "-", config$run_config$sim_name, ".nc") + + FLAREr::update_run_config(lake_directory = lake_directory, + configure_run_file = configure_run_file, + restart_file = restart_file, + start_datetime = start_datetime, + end_datetime = NA, + forecast_start_datetime = forecast_start_datetime, + forecast_horizon = config$run_config$forecast_horizon, + sim_name = config$run_config$sim_name, + site_id = config$location$site_id, + configure_flare = config$run_config$configure_flare, + configure_obs = config$run_config$configure_obs, + use_s3 = config$run_config$use_s3, + bucket = config$s3$restart$bucket, + endpoint = config$s3$restart$endpoint, + use_https = TRUE) + + var1 <- Sys.getenv("AWS_ACCESS_KEY_ID") + var2 <- Sys.getenv("AWS_SECRET_ACCESS_KEY") + Sys.unsetenv("AWS_ACCESS_KEY_ID") + Sys.unsetenv("AWS_SECRET_ACCESS_KEY") + vera4castHelpers::submit(file_name, first_submission = FALSE) + Sys.setenv("AWS_ACCESS_KEY_ID" = var1, + "AWS_SECRET_ACCESS_KEY" = var2) + + noaa_ready <- FLAREr::check_noaa_present(lake_directory, + configure_run_file, + config_set_name = config_set_name) + + reference_date <- lubridate::as_date(forecast_start_datetime) + s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "vera4cast_forecasts", + faasr_prefix = inflow_prefix, + config = config) + avail_dates <- gsub("reference_date=", "", s3$ls()) + inflow_ready <- reference_date %in% avail_dates + } + + invisible(NULL) +} From 198e77ff7e2333c089ccdc676214d62b87abb6ac Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 29 Apr 2026 01:19:52 -0700 Subject: [PATCH 04/44] Fix FCRE callers: pass full path under bare bucket as faasr_prefix/remote_folder --- R/convert_vera4cast_inflow.R | 2 +- R/generate_forecast_score_arrow.R | 4 +++- .../generate_inflow_forecast.R | 2 +- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 19 +++++++++++-------- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/R/convert_vera4cast_inflow.R b/R/convert_vera4cast_inflow.R index e31dc00..6765b03 100644 --- a/R/convert_vera4cast_inflow.R +++ b/R/convert_vera4cast_inflow.R @@ -10,7 +10,7 @@ forecast_df <- NULL for(i in 1:length(variables)){ if(!is.null(config)){ - faasr_prefix <- glue::glue("project_id=vera4cast/duration=P1D/variable={variables[i]}/model_id={model_id}/reference_date={reference_date}") + faasr_prefix <- glue::glue("vera4cast/forecasts/archive-parquet/project_id=vera4cast/duration=P1D/variable={variables[i]}/model_id={model_id}/reference_date={reference_date}") s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "vera4cast_forecasts", faasr_prefix = faasr_prefix, config = config) } else { s3 <- arrow::s3_bucket(bucket = glue::glue("bio230121-bucket01/vera4cast/forecasts/archive-parquet/project_id=vera4cast/duration=P1D/variable={variables[i]}/model_id={model_id}/reference_date={reference_date}"), diff --git a/R/generate_forecast_score_arrow.R b/R/generate_forecast_score_arrow.R index a0d9fc9..e05a020 100644 --- a/R/generate_forecast_score_arrow.R +++ b/R/generate_forecast_score_arrow.R @@ -18,7 +18,9 @@ generate_forecast_score_arrow <- function(targets_df, if(!is.null(config)){ vars <- arrow_env_vars() - output_directory <- FLAREr::flare_arrow_s3_bucket(server_name = "scores", config = config) + output_directory <- FLAREr::flare_arrow_s3_bucket(server_name = "scores", + faasr_prefix = "flare/scores/parquet", + config = config) on.exit(unset_arrow_vars(vars)) }else if(use_s3){ if(is.null(bucket) | is.null(endpoint)){ diff --git a/workflows/glm_aed_flare_rs/generate_inflow_forecast.R b/workflows/glm_aed_flare_rs/generate_inflow_forecast.R index 706dd6e..1c53d8d 100644 --- a/workflows/glm_aed_flare_rs/generate_inflow_forecast.R +++ b/workflows/glm_aed_flare_rs/generate_inflow_forecast.R @@ -13,7 +13,7 @@ if(!is.null(config$s3$vera4cast_targets)){ remote_file = "daily-inflow-targets.csv.gz", server_name = "vera4cast_targets", local_folder = tempdir(), - remote_folder = "project_id=vera4cast/duration=P1D", + remote_folder = "vera4cast/targets/project_id=vera4cast/duration=P1D", config = config) targets_vera <- readr::read_csv(targets_local, show_col_types = FALSE) } else { diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 1a5eee9..6de4ec7 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -26,7 +26,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config_set_name = config_set_name) reference_date <- lubridate::as_date(config$run_config$forecast_start_datetime) - inflow_prefix <- "archive-parquet/project_id=vera4cast/duration=P1D/variable=Temp_C_mean/model_id=inflow_gefsClimAED" + inflow_prefix <- "vera4cast/forecasts/archive-parquet/project_id=vera4cast/duration=P1D/variable=Temp_C_mean/model_id=inflow_gefsClimAED" s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "vera4cast_forecasts", faasr_prefix = inflow_prefix, config = config) @@ -53,7 +53,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", remote_file = "daily-insitu-targets.csv.gz", server_name = "vera4cast_targets", local_folder = tempdir(), - remote_folder = "project_id=vera4cast/duration=P1D", + remote_folder = "vera4cast/targets/project_id=vera4cast/duration=P1D", config = config) readr::read_csv(insitu_local, show_col_types = FALSE) |> @@ -90,8 +90,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", configure_run_file = configure_run_file, config_set_name = config_set_name) - forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", - config = config) + forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", + faasr_prefix = "flare/forecasts/parquet", + config = config) ref_date <- as.character(lubridate::as_date(config$run_config$forecast_start_datetime)) forecast_df <- arrow::open_dataset(forecasts_s3) |> @@ -205,8 +206,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", message("Scoring forecasts") - forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", - config = config) + forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", + faasr_prefix = "flare/forecasts/parquet", + config = config) forecast_df <- arrow::open_dataset(forecasts_s3) |> dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> dplyr::filter(model_id == "glm_aed_flare_v3", @@ -217,8 +219,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", if (config$output_settings$evaluate_past & config$run_config$use_s3) { past_days <- lubridate::as_date(lubridate::as_date(config$run_config$forecast_start_datetime) - lubridate::days(config$run_config$forecast_horizon)) - past_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", - config = config) + past_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", + faasr_prefix = "flare/forecasts/parquet", + config = config) past_forecasts <- arrow::open_dataset(past_s3) |> dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> dplyr::filter(model_id == "glm_aed_flare_v3", From 1d433bb31c84e8c33e0475fe387ce35b4f61a69e Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 29 Apr 2026 22:56:25 -0700 Subject: [PATCH 05/44] Resolve lake_directory via cloned-repo glob; chdir for relative path consistency --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 6de4ec7..37b50b8 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -12,7 +12,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", "AWS_S3_ENDPOINT" = "osn.mghpcc.org", "USE_HTTPS" = TRUE) - lake_directory <- here::here() + cloned <- Sys.glob("/tmp/functions/*/Ashish-Ramrakhiani/FCRE-forecast-code") + lake_directory <- if (length(cloned) > 0) cloned[1] else here::here() + setwd(lake_directory) source(file.path(lake_directory, "R/convert_vera4cast_inflow.R")) source(file.path(lake_directory, "R/generate_forecast_score_arrow.R")) From 962df4b21a1b936f71d35ba63e9bf0bfbe683cac Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 29 Apr 2026 23:10:11 -0700 Subject: [PATCH 06/44] Match tarball-extracted FCRE path (Owner-Repo-{sha}) for lake_directory FaaSr_py uses GitHub tarball API for plain owner/repo entries, which extracts to /tmp/functions/{InvocationID}/{Owner}-{Repo}-{sha}/, not /{Owner}/{Repo}/. The previous glob expected the clone-style path and fell through to here::here() = /action. --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 37b50b8..0285e17 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -12,8 +12,12 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", "AWS_S3_ENDPOINT" = "osn.mghpcc.org", "USE_HTTPS" = TRUE) - cloned <- Sys.glob("/tmp/functions/*/Ashish-Ramrakhiani/FCRE-forecast-code") - lake_directory <- if (length(cloned) > 0) cloned[1] else here::here() + candidates <- c( + Sys.glob("/tmp/functions/*/Ashish-Ramrakhiani-FCRE-forecast-code-*"), + Sys.glob("/tmp/functions/*/Ashish-Ramrakhiani/FCRE-forecast-code") + ) + candidates <- candidates[file.exists(file.path(candidates, "R/convert_vera4cast_inflow.R"))] + lake_directory <- if (length(candidates) > 0) candidates[1] else here::here() setwd(lake_directory) source(file.path(lake_directory, "R/convert_vera4cast_inflow.R")) From c9487398e44c44b98f141836b2887ffa442705ce Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 04:02:19 -0700 Subject: [PATCH 07/44] Use marker-based lake_directory discovery; pin test forecast date and sim_name lake_directory discovery: replace the fork-username-specific glob with a marker-based walk of /tmp/functions for any directory containing configuration//. Works for any fork without code changes; portable across the FaaSr clone vs tarball extraction layouts. configure_run.yml (deployment-only on flare_faasr_int): pin forecast_start_datetime to 2026-05-01 (verified both noaa stage2 and vera4cast inflow_gefsClimAED have data for that date) and rename sim_name to glm_aed_flare_rs_faasr_test so all S3 writes (restart/forecasts/scores) land in a sandbox path distinct from the operational glm_aed_flare_v3 model_id. --- configuration/glm_aed_flare_rs/configure_run.yml | 4 ++-- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 60484de..b1211ca 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -1,9 +1,9 @@ restart_file: .na start_datetime: 2024-10-01 00:00:00 end_datetime: .na -forecast_start_datetime: 2024-12-20 00:00:00 +forecast_start_datetime: 2026-05-01 00:00:00 forecast_horizon: 34 -sim_name: glm_aed_flare_rs +sim_name: glm_aed_flare_rs_faasr_test configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml use_s3: TRUE diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 0285e17..9855258 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -12,12 +12,16 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", "AWS_S3_ENDPOINT" = "osn.mghpcc.org", "USE_HTTPS" = TRUE) - candidates <- c( - Sys.glob("/tmp/functions/*/Ashish-Ramrakhiani-FCRE-forecast-code-*"), - Sys.glob("/tmp/functions/*/Ashish-Ramrakhiani/FCRE-forecast-code") - ) - candidates <- candidates[file.exists(file.path(candidates, "R/convert_vera4cast_inflow.R"))] - lake_directory <- if (length(candidates) > 0) candidates[1] else here::here() + marker <- file.path("configuration", config_set_name, configure_run_file) + candidates <- list.files("/tmp/functions", + pattern = "configure_run\\.yml$", + recursive = TRUE, full.names = TRUE) + candidates <- candidates[grepl(marker, candidates, fixed = TRUE)] + lake_directory <- if (length(candidates) > 0) { + sub(paste0("/", marker, "$"), "", candidates[1]) + } else { + here::here() + } setwd(lake_directory) source(file.path(lake_directory, "R/convert_vera4cast_inflow.R")) From 541cb57805c6ea5cd54d35a53296610b1c176e18 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 05:09:55 -0700 Subject: [PATCH 08/44] Guard top-level vars in generate_inflow_forecast.R against parent-env clobber When sourced from run_fcre_aed_forecast.R, the unconditional lake_directory <- here::here() / config_set_name <- 'glm_aed_flare_v3' re-assignments overwrote the caller's already-resolved values, causing the subsequent set_up_simulation to look in /action/configuration/ glm_aed_flare_v3/configure_run.yml and fail. Wrap each top-level var in 'if (!exists(...))'. Console / RStudio users still get the original defaults (no parent values present); programmatic callers can pre-set lake_directory, config_set_name, configure_run_file, config to avoid the clobber. Backward-compatible. --- workflows/glm_aed_flare_rs/generate_inflow_forecast.R | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/workflows/glm_aed_flare_rs/generate_inflow_forecast.R b/workflows/glm_aed_flare_rs/generate_inflow_forecast.R index 1c53d8d..42084f1 100644 --- a/workflows/glm_aed_flare_rs/generate_inflow_forecast.R +++ b/workflows/glm_aed_flare_rs/generate_inflow_forecast.R @@ -1,9 +1,11 @@ library(tidyverse) -lake_directory <- here::here() -config_set_name <- "glm_aed_flare_v3" -configure_run_file <- "configure_run.yml" -config <- FLAREr::set_up_simulation(configure_run_file,lake_directory, config_set_name = config_set_name) +if (!exists("lake_directory", inherits = FALSE)) lake_directory <- here::here() +if (!exists("config_set_name", inherits = FALSE)) config_set_name <- "glm_aed_flare_v3" +if (!exists("configure_run_file", inherits = FALSE)) configure_run_file <- "configure_run.yml" +if (!exists("config", inherits = FALSE)) { + config <- FLAREr::set_up_simulation(configure_run_file, lake_directory, config_set_name = config_set_name) +} print('read VERA targets...') From c55d8a6ac648b368d4523cbfe6ff5deb1ee13acd Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 05:34:26 -0700 Subject: [PATCH 09/44] source() with local=TRUE to use entry function's env, not stale globalenv faasr_source_r_files at action startup recursively sources every .R file in the FCRE tarball. Many legacy scripts assign top-level lake_directory <- here::here() and config_set_name <- 'ltreb' (etc.) into globalenv. Those values persist. source() defaults to local=FALSE which evaluates in globalenv. So when run_fcre_aed_forecast did source(generate_inflow_forecast.R), the sourced script's exists('lake_directory') check found the stale globalenv value and skipped re-assignment, keeping lake_directory='/action' and config_set_name='ltreb'. The subsequent set_up_simulation tried '/action/configuration/ltreb/configure_aed_run.yml' and failed. Pass local=TRUE to all four source() calls so they evaluate in the entry function's local environment, where lake_directory and config_set_name are correctly set. --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 9855258..0894449 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -24,8 +24,8 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", } setwd(lake_directory) - source(file.path(lake_directory, "R/convert_vera4cast_inflow.R")) - source(file.path(lake_directory, "R/generate_forecast_score_arrow.R")) + source(file.path(lake_directory, "R/convert_vera4cast_inflow.R"), local = TRUE) + source(file.path(lake_directory, "R/generate_forecast_score_arrow.R"), local = TRUE) config <- FLAREr::set_up_simulation(configure_run_file = configure_run_file, lake_directory = lake_directory, @@ -56,7 +56,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", break } - source(file.path(lake_directory, "workflows", config_set_name, "generate_inflow_forecast.R")) + source(file.path(lake_directory, "workflows", config_set_name, "generate_inflow_forecast.R"), local = TRUE) insitu_local <- file.path(tempdir(), "daily-insitu-targets.csv.gz") FLAREr::flare_get_file(local_file = "daily-insitu-targets.csv.gz", @@ -77,7 +77,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", readr::write_csv(file.path(config$file_path$qaqc_data_directory, paste0(config$location$site_id, "-targets-insitu.csv"))) - source(file.path(lake_directory, "workflows", config_set_name, "getLST.R")) + source(file.path(lake_directory, "workflows", config_set_name, "getLST.R"), local = TRUE) data <- get_lst(bbox, config$run_config$start_datetime, config$run_config$forecast_start_datetime) vals <- get_vals(points, data) if (exists("vals") == TRUE) { From ee050b6c92f27a70994afc5350c4637196f8d6f2 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 05:54:28 -0700 Subject: [PATCH 10/44] RFC 3339 dates for rstac and defensive tryCatch around get_lst/get_vals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rstac's STAC datetime parameter requires RFC 3339 format ('YYYY-MM-DDTHH:MM:SSZ'). YAML loads start_datetime/forecast_start_datetime as POSIXct or as space-separated strings, both of which produce 'YYYY-MM-DD HH:MM:SS' on paste() — rstac rejects with 'date time provided not follow the RFC 3339 format'. Format with strftime to ISO 8601 / RFC 3339 before calling get_lst. Wrap get_lst and get_vals in tryCatch so any failure (rstac error, network blip, no images for the date range) falls cleanly into the existing 'No new RS data' placeholder branch instead of crashing the whole forecast iteration. Replaces the brittle exists('vals') check with an explicit !is.null(vals). --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 0894449..823b9ad 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -78,9 +78,16 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", paste0(config$location$site_id, "-targets-insitu.csv"))) source(file.path(lake_directory, "workflows", config_set_name, "getLST.R"), local = TRUE) - data <- get_lst(bbox, config$run_config$start_datetime, config$run_config$forecast_start_datetime) - vals <- get_vals(points, data) - if (exists("vals") == TRUE) { + start_iso <- format(lubridate::as_datetime(config$run_config$start_datetime), "%Y-%m-%dT%H:%M:%SZ", tz = "UTC") + end_iso <- format(lubridate::as_datetime(config$run_config$forecast_start_datetime), "%Y-%m-%dT%H:%M:%SZ", tz = "UTC") + data <- tryCatch( + get_lst(bbox, start_iso, end_iso), + error = function(e) { message("get_lst failed: ", conditionMessage(e)); NULL } + ) + vals <- if (!is.null(data)) { + tryCatch(get_vals(points, data), error = function(e) NULL) + } else NULL + if (!is.null(vals)) { clean_data(vals) |> readr::write_csv(file.path(config$file_path$qaqc_data_directory, paste0(config$location$site_id, "-targets-rs.csv"))) From 22a49215b34d26106f81559ef556e60a9718753b Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 07:02:09 -0700 Subject: [PATCH 11/44] Revert forecast_start_datetime to operational value (2024-12-20) Stage3 met data has duplicate (datetime, parameter, variable) entries on 2025-07-22, 2025-07-24, 2025-07-25 (different prediction values for the same key). With the prior 2026-05-01 pin and start_datetime 2024-10-01, the 19-month spinup window pulled these duplicates into create_met_files's pivot_wider, producing list-columns and crashing mutate(WindSpeed = sqrt(eastward_wind^2 + northward_wind^2)) with "non-numeric argument to binary operator". The 2024-10-01 -> 2024-12-20 window predates the duplicate zone, has both noaa stage2 and vera4cast inflow data available, and runs ~7x faster on spinup. The stage3 duplicate-data issue is an upstream data-quality problem out of scope for this work. --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index b1211ca..f7cb2e1 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -1,7 +1,7 @@ restart_file: .na start_datetime: 2024-10-01 00:00:00 end_datetime: .na -forecast_start_datetime: 2026-05-01 00:00:00 +forecast_start_datetime: 2024-12-20 00:00:00 forecast_horizon: 34 sim_name: glm_aed_flare_rs_faasr_test configure_flare: configure_flare_glm_aed.yml From 5880e147834d8284f415a2d98e7127753d786874 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 08:03:10 -0700 Subject: [PATCH 12/44] Shorten forecast_horizon to 1 day for FaaSr deployment testing Cuts the forward GLM-AED simulation from 34 days x 221 ensembles to 1 day x 221 ensembles. The spinup window (start_datetime -> forecast_start_datetime) is unchanged at 80 days; this only shrinks the forward forecast step. Deployment-only on flare_faasr_int; the operational YAML keeps forecast_horizon: 34. --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index f7cb2e1..5e2393d 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -2,7 +2,7 @@ restart_file: .na start_datetime: 2024-10-01 00:00:00 end_datetime: .na forecast_start_datetime: 2024-12-20 00:00:00 -forecast_horizon: 34 +forecast_horizon: 1 sim_name: glm_aed_flare_rs_faasr_test configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml From dd096af98a98572e5be15d87f6f491ef6fa475dc Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 15:44:26 -0700 Subject: [PATCH 13/44] Add inflation_factor: 1.0 under da_setup for v3.1-dev FLAREr compatibility FLAREr v3.1-dev's generate_initial_conditions.R:128 does init$inflation[] <- config$da_setup$inflation_factor; without the field on the RHS, R errors with "replacement has length zero". The field was introduced in v3.1-dev's param-fitting work and doesn't exist in main, which is why production pipelines running off main don't hit it. 1.0 = no inflation, matching every bundled extdata config in FLAREr itself (default/, aed/, default_pf/, open_meteo/). Harmless on FLAREr main, required on v3.1-dev. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index ebfb12a..d54f870 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -41,6 +41,7 @@ da_setup: no_negative_states: TRUE assimilate_first_step: FALSE use_obs_constraint: TRUE + inflation_factor: 1.0 obs_filename: fcre-targets-rs.csv model_settings: ncore: 4 From 3a408d53189c5ca2f126def5c9f569b266370170 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 15:59:59 -0700 Subject: [PATCH 14/44] Add add_random_noise: 2 under da_setup for v3.1-dev FLAREr compatibility v3.1-dev's run_da_forecast.R branches on config$da_setup$add_random_noise (values 0/1/2) at lines 386, 402, 450, with no default. When the field is missing from the YAML, the if() expression evaluates to a length-zero NULL and R errors with "argument is of length zero". FLAREr main unconditionally calls add_process_noise() per ensemble at run_da_forecast.R:374 -- equivalent to v3.1-dev's add_random_noise == 2 branch. Setting 2 reproduces main's behaviour. Harmless on FLAREr main (field ignored), required on v3.1-dev. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index d54f870..936f408 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -42,6 +42,7 @@ da_setup: assimilate_first_step: FALSE use_obs_constraint: TRUE inflation_factor: 1.0 + add_random_noise: 2 obs_filename: fcre-targets-rs.csv model_settings: ncore: 4 From 2f5e9dd8183bc985a613a54188228f8432595dc8 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 16:15:49 -0700 Subject: [PATCH 15/44] Add depth_sd: 0.0 under model_settings for v3.1-dev FLAREr compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v3.1-dev's run_flare.R:145 reads config$model_settings$depth_sd into states_non_vertical$depth_sd. Then run_da_forecast.R:388 / :446 use it as the sd argument to rnorm() to perturb lake_depth per ensemble per timestep. Without the YAML field, sd=NULL and rnorm errors with 'invalid arguments'. FLAREr main has no states_non_vertical concept and no depth perturbation in DA — depth_sd: 0.0 reproduces that behavior (rnorm(n, mean, sd=0) returns mean exactly, no randomness). Harmless on main, required on v3.1-dev. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index 936f408..a71945a 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -57,6 +57,7 @@ model_settings: obs_config_file: observations_config_aed.csv states_config_file: states_config_aed.csv depth_model_sd_config_file: depth_model_sd_aed.csv + depth_sd: 0.0 default_init: lake_depth: 9.4 #not a modeled state temp: [25.667, 24.9101, 23.067, 21.8815, 19.6658, 16.5739, 12.9292, 12.8456, 12.8127, 12.8079, 12.778] From 88b67c53345b086f831519b898c339c4b7015737 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 16:27:49 -0700 Subject: [PATCH 16/44] Reduce spinup to 3 days and ensemble to 31 for fast deployment testing start_datetime: 2024-12-17 (was 2024-10-01) -- DA loop runs 3 daily iterations instead of 80, exercising the same code path with ~25x less spinup compute. ensemble_size: 31 (was 221) -- each DA / forecast step runs 31 GLM-AED simulations instead of 221, ~7x less per-step compute. Combined: ~175x compute reduction for the GLM-AED-heavy DA loop. Deployment-only change on flare_faasr_int; the operational YAML in the upstream branch keeps start_datetime: 2024-10-01 and ensemble_size: 221. The test only verifies pipeline wiring, not forecast quality -- a 3-day spinup is not enough for AED state convergence. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 2 +- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index a71945a..d810861 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -36,7 +36,7 @@ location: da_setup: da_method: enkf par_fit_method: perturb - ensemble_size: 221 + ensemble_size: 31 localization_distance: .na #distance in meters were covariances in the model error are used no_negative_states: TRUE assimilate_first_step: FALSE diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 5e2393d..9d38451 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -1,5 +1,5 @@ restart_file: .na -start_datetime: 2024-10-01 00:00:00 +start_datetime: 2024-12-17 00:00:00 end_datetime: .na forecast_start_datetime: 2024-12-20 00:00:00 forecast_horizon: 1 From 1514ec6b000ee1cc6b112009a06c083a23a787db Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 18:13:44 -0700 Subject: [PATCH 17/44] Drive forecast model_id filter from sim_name instead of hardcoded glm_aed_flare_v3 Three filter sites in run_fcre_aed_forecast.R hardcoded model_id == 'glm_aed_flare_v3' (carried over from combined_run_aed.R). Two consequences when sim_name != 'glm_aed_flare_v3': 1. Reads the production forecast instead of the one we just wrote -- the pipeline doesn't actually exercise its own output. 2. Score parquet auto-partitions by the model_id column from the filtered data, so writes land at .../scores/.../model_id=glm_aed_flare_v3/... -- additive pollution of the production namespace. Replace the literal with config$run_config$sim_name at all three sites. The script now reads the forecast it just produced, scores it, and writes scores under its own sim_name; generic for any sim_name. --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 823b9ad..06199e8 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -113,7 +113,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", ref_date <- as.character(lubridate::as_date(config$run_config$forecast_start_datetime)) forecast_df <- arrow::open_dataset(forecasts_s3) |> - filter(model_id == "glm_aed_flare_v3", + filter(model_id == config$run_config$sim_name, site_id == "fcre", reference_date == ref_date) |> collect() |> @@ -228,7 +228,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config) forecast_df <- arrow::open_dataset(forecasts_s3) |> dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> - dplyr::filter(model_id == "glm_aed_flare_v3", + dplyr::filter(model_id == config$run_config$sim_name, site_id == config$location$site_id, reference_date == lubridate::as_datetime(config$run_config$forecast_start_datetime)) |> dplyr::collect() @@ -241,7 +241,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config) past_forecasts <- arrow::open_dataset(past_s3) |> dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> - dplyr::filter(model_id == "glm_aed_flare_v3", + dplyr::filter(model_id == config$run_config$sim_name, site_id == config$location$site_id, reference_date == past_days) |> dplyr::collect() From 9d9b57538f018be2476363f37724570c0cbc9488 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sat, 2 May 2026 23:36:22 -0700 Subject: [PATCH 18/44] tryCatch around vera4castHelpers::submit so submit errors don't kill the run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Submit can fail for transient reasons (vera4cast API down, rate limit, schema validation) that aren't related to whether our forecast pipeline worked. Catch and log instead of crashing — restart/forecast/score parquet writes already succeeded at this point, the run should reach completion regardless. AWS env restore now runs after either branch (submit success or caught error). --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 06199e8..bf3849e 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -290,7 +290,10 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", var2 <- Sys.getenv("AWS_SECRET_ACCESS_KEY") Sys.unsetenv("AWS_ACCESS_KEY_ID") Sys.unsetenv("AWS_SECRET_ACCESS_KEY") - vera4castHelpers::submit(file_name, first_submission = FALSE) + tryCatch( + vera4castHelpers::submit(file_name, first_submission = FALSE), + error = function(e) message("vera4cast submit failed (continuing): ", conditionMessage(e)) + ) Sys.setenv("AWS_ACCESS_KEY_ID" = var1, "AWS_SECRET_ACCESS_KEY" = var2) From f0aa23d07dbe16d40c5a963bfab610cfe4ca15b4 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sun, 3 May 2026 00:00:52 -0700 Subject: [PATCH 19/44] Narrow arrow prefix to specific partition; pass config to update_run_config Two fixes: 1. arrow::open_dataset partition listing was taking ~65min each over OSN because the prefix was the bucket root (flare/forecasts/parquet) and arrow had to enumerate every model_id x reference_date directory under the entire forecasts bucket to build the schema, even though the dplyr filter only wanted one partition. Push the partition keys into the prefix so arrow opens exactly one parquet file. The dplyr filter is now redundant and removed. Two sites (vera4cast aggregation block and score block, plus the past_forecasts variant). 2. update_run_config in v3.1-dev FLAREr has a config parameter not present in main. Internally it calls flare_put_file(config = config). Our caller wasn't passing config, so the inner call errored with 'argument config is missing'. Add config = config to the call. --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index bf3849e..0b4778f 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -107,15 +107,16 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", configure_run_file = configure_run_file, config_set_name = config_set_name) - forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", - faasr_prefix = "flare/forecasts/parquet", - config = config) - ref_date <- as.character(lubridate::as_date(config$run_config$forecast_start_datetime)) + forecasts_s3 <- FLAREr::flare_arrow_s3_bucket( + server_name = "forecasts_parquet", + faasr_prefix = paste0("flare/forecasts/parquet", + "/site_id=", config$location$site_id, + "/model_id=", config$run_config$sim_name, + "/reference_date=", ref_date), + config = config + ) forecast_df <- arrow::open_dataset(forecasts_s3) |> - filter(model_id == config$run_config$sim_name, - site_id == "fcre", - reference_date == ref_date) |> collect() |> mutate(datetime = lubridate::as_datetime(datetime)) @@ -223,27 +224,31 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", message("Scoring forecasts") - forecasts_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", - faasr_prefix = "flare/forecasts/parquet", - config = config) + forecasts_s3 <- FLAREr::flare_arrow_s3_bucket( + server_name = "forecasts_parquet", + faasr_prefix = paste0("flare/forecasts/parquet", + "/site_id=", config$location$site_id, + "/model_id=", config$run_config$sim_name, + "/reference_date=", as.character(lubridate::as_date(config$run_config$forecast_start_datetime))), + config = config + ) forecast_df <- arrow::open_dataset(forecasts_s3) |> dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> - dplyr::filter(model_id == config$run_config$sim_name, - site_id == config$location$site_id, - reference_date == lubridate::as_datetime(config$run_config$forecast_start_datetime)) |> dplyr::collect() if (config$output_settings$evaluate_past & config$run_config$use_s3) { past_days <- lubridate::as_date(lubridate::as_date(config$run_config$forecast_start_datetime) - lubridate::days(config$run_config$forecast_horizon)) - past_s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "forecasts_parquet", - faasr_prefix = "flare/forecasts/parquet", - config = config) + past_s3 <- FLAREr::flare_arrow_s3_bucket( + server_name = "forecasts_parquet", + faasr_prefix = paste0("flare/forecasts/parquet", + "/site_id=", config$location$site_id, + "/model_id=", config$run_config$sim_name, + "/reference_date=", as.character(past_days)), + config = config + ) past_forecasts <- arrow::open_dataset(past_s3) |> dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> - dplyr::filter(model_id == config$run_config$sim_name, - site_id == config$location$site_id, - reference_date == past_days) |> dplyr::collect() } else { past_forecasts <- NULL @@ -284,6 +289,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", use_s3 = config$run_config$use_s3, bucket = config$s3$restart$bucket, endpoint = config$s3$restart$endpoint, + config = config, use_https = TRUE) var1 <- Sys.getenv("AWS_ACCESS_KEY_ID") From 59c962c187b90c63765b9e63c08595a98827d3d2 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sun, 3 May 2026 00:17:27 -0700 Subject: [PATCH 20/44] Synthesize partition columns after collect() so downstream dplyr keys still resolve Side effect of pointing arrow::open_dataset directly at a leaf partition path (site_id=X/model_id=Y/reference_date=Z): partition columns are embedded in the path, not in the parquet file. arrow only synthesizes them when discovering partitions from the bucket root. At the leaf path, the resulting tibble has only the parquet's own columns - no site_id, model_id, reference_date. Downstream dplyr summarize(.by = c(..., site_id, model_id, ...)) and write_dataset(partitioning = c('site_id','model_id','reference_date')) both need those columns. Add them back via mutate() with the values we already have (the same values we used to construct the prefix). Three sites: vera4cast aggregation forecast_df, score block forecast_df, score block past_forecasts. --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 0b4778f..5fbf979 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -118,7 +118,10 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", ) forecast_df <- arrow::open_dataset(forecasts_s3) |> collect() |> - mutate(datetime = lubridate::as_datetime(datetime)) + mutate(datetime = lubridate::as_datetime(datetime), + site_id = config$location$site_id, + model_id = config$run_config$sim_name, + reference_date = ref_date) vera_variables <- c("Temp_C_mean","Chla_ugL_mean", "DO_mgL_mean", "fDOM_QSU_mean", "NH4_ugL_sample", "NO3NO2_ugL_sample", "SRP_ugL_sample", "DIC_mgL_sample","Secchi_m_sample", @@ -233,8 +236,10 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config ) forecast_df <- arrow::open_dataset(forecasts_s3) |> - dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> - dplyr::collect() + dplyr::collect() |> + dplyr::mutate(site_id = config$location$site_id, + model_id = config$run_config$sim_name, + reference_date = lubridate::as_date(config$run_config$forecast_start_datetime)) if (config$output_settings$evaluate_past & config$run_config$use_s3) { past_days <- lubridate::as_date(lubridate::as_date(config$run_config$forecast_start_datetime) - @@ -248,8 +253,10 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config ) past_forecasts <- arrow::open_dataset(past_s3) |> - dplyr::mutate(reference_date = lubridate::as_date(reference_date)) |> - dplyr::collect() + dplyr::collect() |> + dplyr::mutate(site_id = config$location$site_id, + model_id = config$run_config$sim_name, + reference_date = past_days) } else { past_forecasts <- NULL } From ef79553a8e4a634d399c07d74914d8a38e9860e2 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sun, 3 May 2026 02:46:35 -0700 Subject: [PATCH 21/44] Deployment-only: clean_start=TRUE in set_up_simulation for repeatable testing Force get_run_config to ignore S3-leftover configure_run.yml and use local YAML on every invoke. Without this, each test run inherits the prior run's advanced forecast_start_datetime + restart_file pointer and tries to resume -- which exposes latent bugs in loop logic that don't manifest in a daily-cron production setting where each invoke runs a single iteration. Revert before merging to main: production runs rely on clean_start=FALSE (default) so the daily restart chain works. --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 5fbf979..63ecc64 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -29,7 +29,8 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config <- FLAREr::set_up_simulation(configure_run_file = configure_run_file, lake_directory = lake_directory, - config_set_name = config_set_name) + config_set_name = config_set_name, + clean_start = TRUE) noaa_ready <- FLAREr::check_noaa_present(lake_directory, configure_run_file, From d55e77a0b5f894b95cebb7998df45f74b243c577 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Sun, 3 May 2026 02:50:48 -0700 Subject: [PATCH 22/44] Revert clean_start=TRUE override Reverting the deployment-only override added in 045470e. Test repeatability will be handled by clearing the test sandbox's S3 state before each invoke instead of skipping the S3-resume branch in code. The set_up_simulation call returns to its default (clean_start=FALSE). --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 63ecc64..5fbf979 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -29,8 +29,7 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config <- FLAREr::set_up_simulation(configure_run_file = configure_run_file, lake_directory = lake_directory, - config_set_name = config_set_name, - clean_start = TRUE) + config_set_name = config_set_name) noaa_ready <- FLAREr::check_noaa_present(lake_directory, configure_run_file, From 4d1873f300ef57fb5ed139f5307e7ab43de38ed5 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Tue, 5 May 2026 17:09:14 -0700 Subject: [PATCH 23/44] Pin forecast date to data edge so loop exits cleanly after iter 1 forecast_start_datetime: 2026-05-04 -- latest date with both noaa stage2 and vera4cast inflow_gefsClimAED data available. After iter 1 update_run_config advances to 2026-05-05; no data published for that date yet, so noaa_ready/inflow_ready flip FALSE and the while loop exits naturally. Mirrors the daily-cron production pattern where each cron firing runs exactly one iteration. start_datetime: 2026-05-01 -- 3-day spinup, well clear of the 2025-07-22 to 2025-07-25 stage3 met-data duplicate zone. --- configuration/glm_aed_flare_rs/configure_run.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 9d38451..5d70d01 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -1,7 +1,7 @@ restart_file: .na -start_datetime: 2024-12-17 00:00:00 +start_datetime: 2026-05-01 00:00:00 end_datetime: .na -forecast_start_datetime: 2024-12-20 00:00:00 +forecast_start_datetime: 2026-05-04 00:00:00 forecast_horizon: 1 sim_name: glm_aed_flare_rs_faasr_test configure_flare: configure_flare_glm_aed.yml From 864b1d2d8fc3fdd116b691de38a3042d19051adc Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 6 May 2026 21:48:35 -0700 Subject: [PATCH 24/44] Drop dead duplicate arrow::open_dataset block after tryCatch --- R/convert_vera4cast_inflow.R | 5 ----- 1 file changed, 5 deletions(-) diff --git a/R/convert_vera4cast_inflow.R b/R/convert_vera4cast_inflow.R index 6765b03..884d268 100644 --- a/R/convert_vera4cast_inflow.R +++ b/R/convert_vera4cast_inflow.R @@ -33,11 +33,6 @@ for(i in 1:length(variables)){ }) - # df <- arrow::open_dataset(s3) |> - # dplyr::filter(site_id == "tubr") |> - # dplyr::collect() |> - # dplyr::mutate(variable = variables[i]) - forecast_df <- dplyr::bind_rows(forecast_df, df) } From fce5f9e7dede7ebd2be0bdf300e9ec9fc7dc2cc1 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 6 May 2026 23:48:22 -0700 Subject: [PATCH 25/44] Refresh in-memory config after update_run_config inside loop After update_run_config writes the advanced forecast_start_datetime to disk + S3, the outer-scope config in run_fcre_aed_forecast still holds the previous values. Sourced helpers on the next iteration (generate_inflow_forecast.R -> convert_vera4cast_inflow) read the stale value and write inflow partitions under the previous reference_date, while run_flare re-reads config from disk and looks under the new one -- producing arrow IOError 'No such file or directory' on the reference_date partition. Refreshing config via set_up_simulation right after update_run_config keeps the in-memory view aligned with on-disk state. combined_run_aed.R doesn't trip this because each cron firing runs the loop exactly once per process; this script runs multiple iterations in one process. --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 5fbf979..2a71d64 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -299,6 +299,21 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config, use_https = TRUE) + # POSSIBLE BUG FIX -- safe to remove if upstream behavior changes. + # update_run_config just wrote a new forecast_start_datetime to + # disk + S3, but the in-memory `config` still holds the previous + # values. Sourced helpers on the next iteration + # (generate_inflow_forecast.R -> convert_vera4cast_inflow) read + # from this outer `config` and would write inflow partitions under + # the previous reference_date, while run_flare re-reads config + # from disk and looks under the new one -- producing an + # arrow::open_dataset IOError. Refreshing here keeps the in-memory + # view aligned with on-disk state. combined_run_aed.R doesn't need + # this because each cron firing runs the loop exactly once. + config <- FLAREr::set_up_simulation(configure_run_file = configure_run_file, + lake_directory = lake_directory, + config_set_name = config_set_name) + var1 <- Sys.getenv("AWS_ACCESS_KEY_ID") var2 <- Sys.getenv("AWS_SECRET_ACCESS_KEY") Sys.unsetenv("AWS_ACCESS_KEY_ID") From 78277d6df1b48c3aee956da4e1703d328b2c91b8 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 7 May 2026 00:37:53 -0700 Subject: [PATCH 26/44] Extend test spinup to 4 days so chained iterations resolve restart_index Iter 1 produces a restart NetCDF whose time vector spans [start_datetime, forecast_start_datetime + horizon]. After iter 1, the combined_run_aed.R-style carry-over math sets the next iteration's start_datetime to old_forecast_start_datetime - 4 days. With a 3-day spinup that target lands BEFORE the NetCDF's earliest time, so which(datetime == new_sd) returns integer(0) and generate_initial_conditions crashes with 'replacement has length zero' when indexing pars_restart with the empty restart_index. Bumping start_datetime from 2026-05-01 to 2026-04-30 gives a 4-day spinup; iter 1's NetCDF then includes 2026-04-30, so iter 2 finds its restart_index. Production daily-cron config doesn't trip this because its 80-day spinup leaves plenty of cushion. --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 5d70d01..ee2b484 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -1,5 +1,5 @@ restart_file: .na -start_datetime: 2026-05-01 00:00:00 +start_datetime: 2026-04-30 00:00:00 end_datetime: .na forecast_start_datetime: 2026-05-04 00:00:00 forecast_horizon: 1 From e3dd5b8ad11259653bbafcd603b0c7484e925d79 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 7 May 2026 02:52:10 -0700 Subject: [PATCH 27/44] Wrap check_noaa_present calls so missing-partition error exits loop cleanly FLAREr::check_noaa_present catches arrow IOError on a missing NOAA partition with a tryCatch whose handler calls message(error_message), passing the simpleError condition object. message() with a single condition argument signals the condition as-is, preserving its error class -- so the FaaSr executor's outer withCallingHandlers(error=) catches it and marks the action as failed instead of letting the function return FALSE. The team's daily-cron pattern never trips this because their config runs against past dates where stage2 NOAA is always present. Our chained-iteration pattern eventually advances past the published horizon and hits the catch arm. Wrap the two call sites with our own tryCatch that converts any error into noaa_ready=FALSE so the while-loop exits cleanly. Remove this workaround when upstream check_noaa_present is fixed to use message(conditionMessage(e)). --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 2a71d64..ccdf8d8 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -31,9 +31,21 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", lake_directory = lake_directory, config_set_name = config_set_name) - noaa_ready <- FLAREr::check_noaa_present(lake_directory, - configure_run_file, - config_set_name = config_set_name) + # WORKAROUND: FLAREr::check_noaa_present catches arrow IOError when the + # NOAA partition is missing, but its handler does `message(error_message)` + # with the condition object, which re-signals the error and aborts the + # action instead of returning FALSE. Drop this wrapper once FLAREr's + # check_noaa_present uses message(conditionMessage(e)). + noaa_ready <- tryCatch( + FLAREr::check_noaa_present(lake_directory, + configure_run_file, + config_set_name = config_set_name), + error = function(e) { + message("check_noaa_present errored (treating as not-ready): ", + conditionMessage(e)) + FALSE + } + ) reference_date <- lubridate::as_date(config$run_config$forecast_start_datetime) inflow_prefix <- "vera4cast/forecasts/archive-parquet/project_id=vera4cast/duration=P1D/variable=Temp_C_mean/model_id=inflow_gefsClimAED" @@ -325,9 +337,17 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", Sys.setenv("AWS_ACCESS_KEY_ID" = var1, "AWS_SECRET_ACCESS_KEY" = var2) - noaa_ready <- FLAREr::check_noaa_present(lake_directory, - configure_run_file, - config_set_name = config_set_name) + # See note above on the initial call site. + noaa_ready <- tryCatch( + FLAREr::check_noaa_present(lake_directory, + configure_run_file, + config_set_name = config_set_name), + error = function(e) { + message("check_noaa_present errored (treating as not-ready): ", + conditionMessage(e)) + FALSE + } + ) reference_date <- lubridate::as_date(forecast_start_datetime) s3 <- FLAREr::flare_arrow_s3_bucket(server_name = "vera4cast_forecasts", From 1b136f0b3a5a22f871876eb4473c1c9fba3ca1d5 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 8 May 2026 09:02:38 -0700 Subject: [PATCH 28/44] Demo: glm_aed_flare_rs_demo_faasr sandbox config for FaaSr-mode run --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index ee2b484..3efd68c 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -3,7 +3,7 @@ start_datetime: 2026-04-30 00:00:00 end_datetime: .na forecast_start_datetime: 2026-05-04 00:00:00 forecast_horizon: 1 -sim_name: glm_aed_flare_rs_faasr_test +sim_name: glm_aed_flare_rs_demo_faasr configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml use_s3: TRUE From 9f16341558d6ab87d972f0c94d3ebd3982ffc089 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 14 May 2026 22:01:12 -0700 Subject: [PATCH 29/44] Defensive GLM_PATH default; pass local_path for mode=local reads Only set GLM_PATH='GLM3r' when unset, so a container ENV (or any explicit pre-set value) wins. Pass local_path to flare_arrow_s3_bucket at the two forecasts_parquet read sites so the read location matches where write_forecast writes in local mode. --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index ccdf8d8..b2b8c12 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -5,7 +5,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", library(lubridate) set.seed(100) - Sys.setenv('GLM_PATH' = 'GLM3r') + # Respect any pre-set GLM_PATH (e.g. container ENV in FaaSr mode); + # fall back to GLM3r for local/S3 runs that don't set it explicitly. + if (Sys.getenv("GLM_PATH") == "") Sys.setenv("GLM_PATH" = "GLM3r") options(future.globals.maxSize = 891289600) Sys.setenv("AWS_DEFAULT_REGION" = "amnh1", @@ -126,6 +128,10 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", "/site_id=", config$location$site_id, "/model_id=", config$run_config$sim_name, "/reference_date=", ref_date), + local_path = file.path(lake_directory, "forecasts", "parquet", + paste0("site_id=", config$location$site_id), + paste0("model_id=", config$run_config$sim_name), + paste0("reference_date=", ref_date)), config = config ) forecast_df <- arrow::open_dataset(forecasts_s3) |> @@ -245,6 +251,10 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", "/site_id=", config$location$site_id, "/model_id=", config$run_config$sim_name, "/reference_date=", as.character(lubridate::as_date(config$run_config$forecast_start_datetime))), + local_path = file.path(lake_directory, "forecasts", "parquet", + paste0("site_id=", config$location$site_id), + paste0("model_id=", config$run_config$sim_name), + paste0("reference_date=", as.character(lubridate::as_date(config$run_config$forecast_start_datetime)))), config = config ) forecast_df <- arrow::open_dataset(forecasts_s3) |> From 0d6f314ef01976ad0dbaba7f9d39ef2425912534 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 14 May 2026 22:56:09 -0700 Subject: [PATCH 30/44] Set base_AED_phyto_pars_nml_file for v3.1-dev-netcdf-v2 FLAREr's run_flare on v3.1-dev-netcdf-v2 reads config$model_settings$base_AED_phyto_pars_nml_file (with _file suffix) and passes it to update_phy_states_obs_mapping, which calls read_nml on it. Point the new field at aed2.nml, which contains the aed_phytoplankton block the function expects. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index d810861..583f8ca 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -50,6 +50,7 @@ model_settings: base_GLM_nml: glm3.nml base_AED_nml: aed2.nml base_AED_phyto_pars_nml: aed_phyto_pars.csv + base_AED_phyto_pars_nml_file: aed2.nml base_AED_zoop_pars_nml: aed2_zoop_pars.nml max_model_layers: 100 modeled_depths: [0.00,1.00,1.60,2.00,3.00,4.00,5.00,6.00,7.00,8.00,9.00] From 0647ab149dc23b723dc454ca07b77859c94bfde1 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 14 May 2026 23:08:15 -0700 Subject: [PATCH 31/44] Add init_restart_from_file/fname placeholders to glm3.nml FLAREr's run_model on v3.1-dev-netcdf-v2 sets init_restart_from_file via glmtools::set_nml() before each GLM run; the call requires the variable to exist in the base nml. Adds the placeholder (=0) plus init_restart_fname to match the bundled aed/default templates. --- configuration/glm_aed_flare_rs/glm3.nml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/glm_aed_flare_rs/glm3.nml b/configuration/glm_aed_flare_rs/glm3.nml index 41a188b..a2489de 100755 --- a/configuration/glm_aed_flare_rs/glm3.nml +++ b/configuration/glm_aed_flare_rs/glm3.nml @@ -73,6 +73,8 @@ avg_surf_temp = 6 restart_variables = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 restart_mixer_count = 0 + init_restart_fname = 'glm_restart.nc' + init_restart_from_file = 0 / &meteorology met_sw = .true. From cea5946d66aef55d62928ae2bbd47101cb4903df Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 14 May 2026 23:17:58 -0700 Subject: [PATCH 32/44] Add restart_fname placeholder under &output Last NML field FLAREr's run_model writes via glmtools::set_nml that wasn't present in glm3.nml. Mirrors the bundled aed/default templates. --- configuration/glm_aed_flare_rs/glm3.nml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/glm_aed_flare_rs/glm3.nml b/configuration/glm_aed_flare_rs/glm3.nml index a2489de..b9ba009 100755 --- a/configuration/glm_aed_flare_rs/glm3.nml +++ b/configuration/glm_aed_flare_rs/glm3.nml @@ -57,6 +57,8 @@ out_fn = "output" nsave = 1 csv_lake_fname = 'lake' + restart_fname = 'glm_restart.nc' + restart_nsave = 1 / &init_profiles num_heights = 28 From 3e8985ca9a20db561a17bb6730c017ee49507c7b Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Thu, 14 May 2026 23:35:14 -0700 Subject: [PATCH 33/44] Resolve GLM_PATH to embedded container binary when available Previous defensive default could land on GLM3r in the FaaSr container, which doesn't have GLM3r installed. Check for /opt/glm/glm first; that's where the FaaSr container drops the embedded GLM binary. Log the resolved value at function entry to make the chain transparent. --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index b2b8c12..b893a3f 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -5,9 +5,20 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", library(lubridate) set.seed(100) - # Respect any pre-set GLM_PATH (e.g. container ENV in FaaSr mode); - # fall back to GLM3r for local/S3 runs that don't set it explicitly. - if (Sys.getenv("GLM_PATH") == "") Sys.setenv("GLM_PATH" = "GLM3r") + # Resolve GLM_PATH: respect any pre-set value (container ENV or caller's + # Sys.setenv); else use the container's embedded binary if present; else + # fall back to GLM3r for local/S3 runs. + glm_path_initial <- Sys.getenv("GLM_PATH") + message("GLM_PATH at function entry: '", glm_path_initial, "'") + if (glm_path_initial == "" || glm_path_initial == "GLM3r") { + if (file.exists("/opt/glm/glm")) { + Sys.setenv("GLM_PATH" = "/opt/glm/glm") + message("GLM_PATH resolved to container binary: /opt/glm/glm") + } else if (glm_path_initial == "") { + Sys.setenv("GLM_PATH" = "GLM3r") + message("GLM_PATH resolved to GLM3r (no container binary found)") + } + } options(future.globals.maxSize = 891289600) Sys.setenv("AWS_DEFAULT_REGION" = "amnh1", From 6f556caa095eb90eae42e66c5600c45d2052415e Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 15 May 2026 00:43:29 -0700 Subject: [PATCH 34/44] Point obs_filename at insitu file instead of remote-sensing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fcre-targets-rs.csv is the remote-sensing (LST) target file — usually empty because cloud-free imagery is sparse. With it as obs_filename, create_obs_matrix returned all-NA observations, obs_count was 0 at every iter, and FLAREr's no-DA branch fired — which has a separate uninitialized-pars_corr bug. Using fcre-targets-insitu.csv (the rich daily-depth observations) matches Quinn's bundled aed config and keeps every iter on the DA branch. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index 583f8ca..9a5631d 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -43,7 +43,7 @@ da_setup: use_obs_constraint: TRUE inflation_factor: 1.0 add_random_noise: 2 - obs_filename: fcre-targets-rs.csv + obs_filename: fcre-targets-insitu.csv model_settings: ncore: 4 model_name: glm_aed From 01e7ede6a656946ab972bae0b4d9e1bad424ebbd Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 15 May 2026 01:44:04 -0700 Subject: [PATCH 35/44] Drop secchi row from observations_config_aed.csv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was the only non-vertical entry (multi_depth=0). v3.1-dev-netcdf-v2 adds a new code path for non-vertical obs that requires model_source / model_variable / model_depth_m columns plus a non_vertical_noise_config csv — neither of which our config has. Without those, extract_modeled errors with 'missing value where TRUE/FALSE needed'. Removing secchi sidesteps the path entirely and matches what the rest of the FaaSr test workflow exercises. --- configuration/glm_aed_flare_rs/observations_config_aed.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/observations_config_aed.csv b/configuration/glm_aed_flare_rs/observations_config_aed.csv index 2a7d61c..54c77d7 100644 --- a/configuration/glm_aed_flare_rs/observations_config_aed.csv +++ b/configuration/glm_aed_flare_rs/observations_config_aed.csv @@ -3,5 +3,4 @@ temp,degC,1,Temp_C_mean,1 OXY_oxy,mmol m-3,15,DO_mgL_mean,1 OGM_doc_total,mmol m-3,18.7,fDOM_QSU_mean,1 PHY_TCHLA,ugL,2,Chla_ugL_mean,1 -secchi, unitless,0.25, Secchi_m_sample,0 From f71af21a3051a9ec5e0fb8548e8cffaf499c85c1 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 15 May 2026 11:47:33 -0700 Subject: [PATCH 36/44] Adopt non-vertical config bundle from FLAREr v3.1-dev-netcdf-v2 Three files brought in from FLARE-forecast/FLAREr@v3.1-dev-netcdf-v2's inst/extdata/configuration/aed/ (per Quinn's reference): - non_vertical_noise_config.csv (new file, verbatim from upstream) - non_vertical_noise_config_file key added to configure_flare_glm_aed.yml pointing at the new file - observations_config_aed.csv schema extended with model_source, model_variable, model_depth_m columns. Existing multi_depth=1 rows get NA in the new columns (no behavior change). secchi row restored using upstream's diagnostic/extc_coeff/1.0 pattern but keeping FCRE's Secchi_m_sample target name. --- .../glm_aed_flare_rs/configure_flare_glm_aed.yml | 1 + .../glm_aed_flare_rs/non_vertical_noise_config.csv | 4 ++++ .../glm_aed_flare_rs/observations_config_aed.csv | 12 ++++++------ 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 configuration/glm_aed_flare_rs/non_vertical_noise_config.csv diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index 9a5631d..0974556 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -58,6 +58,7 @@ model_settings: obs_config_file: observations_config_aed.csv states_config_file: states_config_aed.csv depth_model_sd_config_file: depth_model_sd_aed.csv + non_vertical_noise_config_file: non_vertical_noise_config.csv depth_sd: 0.0 default_init: lake_depth: 9.4 #not a modeled state diff --git a/configuration/glm_aed_flare_rs/non_vertical_noise_config.csv b/configuration/glm_aed_flare_rs/non_vertical_noise_config.csv new file mode 100644 index 0000000..0f83763 --- /dev/null +++ b/configuration/glm_aed_flare_rs/non_vertical_noise_config.csv @@ -0,0 +1,4 @@ +model_variable,model_source,model_depth_m,process_noise_sd +lake_depth,state,NA,0.01 +extc,diagnostic,1.0,NA +CAR_ch4_atm,diagnostic_daily,NA,0.0 \ No newline at end of file diff --git a/configuration/glm_aed_flare_rs/observations_config_aed.csv b/configuration/glm_aed_flare_rs/observations_config_aed.csv index 54c77d7..7e2f7dd 100644 --- a/configuration/glm_aed_flare_rs/observations_config_aed.csv +++ b/configuration/glm_aed_flare_rs/observations_config_aed.csv @@ -1,6 +1,6 @@ -state_names_obs,obs_units,obs_sd,target_variable, multi_depth -temp,degC,1,Temp_C_mean,1 -OXY_oxy,mmol m-3,15,DO_mgL_mean,1 -OGM_doc_total,mmol m-3,18.7,fDOM_QSU_mean,1 -PHY_TCHLA,ugL,2,Chla_ugL_mean,1 - +state_names_obs,obs_units,obs_sd,target_variable,multi_depth,model_source,model_variable,model_depth_m +temp,degC,1,Temp_C_mean,1,NA,NA,NA +OXY_oxy,mmol m-3,15,DO_mgL_mean,1,NA,NA,NA +OGM_doc_total,mmol m-3,18.7,fDOM_QSU_mean,1,NA,NA,NA +PHY_TCHLA,ugL,2,Chla_ugL_mean,1,NA,NA,NA +secchi,unitless,0.25,Secchi_m_sample,0,diagnostic,extc_coeff,1.0 From 5df395bb797a0bbee2518a4b9475c7f3300a0dba Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 15 May 2026 13:09:48 -0700 Subject: [PATCH 37/44] Use run_flare's actual restart filename instead of reconstructing it write_restart on v3.1-dev-netcdf-v2 returns .zip when GLM restart staging produces extra files (the default for this branch) and .nc otherwise. Hard-coding .nc in the next iteration's restart_file caused get_restart_file to 404 on iter 2 because iter 1 had uploaded the file as .zip. Capturing run_flare's return value and using basename() on whatever it actually wrote keeps the chain consistent regardless of which format the underlying write produced. --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index b893a3f..fee8c98 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -128,9 +128,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", paste0(config$location$site_id, "-targets-rs.csv"))) } - FLAREr::run_flare(lake_directory = lake_directory, - configure_run_file = configure_run_file, - config_set_name = config_set_name) + flare_result <- FLAREr::run_flare(lake_directory = lake_directory, + configure_run_file = configure_run_file, + config_set_name = config_set_name) ref_date <- as.character(lubridate::as_date(config$run_config$forecast_start_datetime)) forecasts_s3 <- FLAREr::flare_arrow_s3_bucket( @@ -311,9 +311,12 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", forecast_start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) + lubridate::days(1) start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) - lubridate::days(4) - restart_file <- paste0(config$location$site_id, "-", - (lubridate::as_date(forecast_start_datetime) - days(1)), - "-", config$run_config$sim_name, ".nc") + # Use the actual filename run_flare wrote, instead of reconstructing it. + # write_restart returns .zip when GLM restart staging is enabled (Quinn's + # v3.1-dev-netcdf-v2 default) and .nc otherwise. Hard-coding either + # extension would mismatch what was uploaded to S3, so get_restart_file + # would 404 on the next iteration. + restart_file <- basename(flare_result$restart_file) FLAREr::update_run_config(lake_directory = lake_directory, configure_run_file = configure_run_file, From b736dead20bb06744152a62d9ec5a61dcff59199 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 15 May 2026 13:28:32 -0700 Subject: [PATCH 38/44] Save restart at every timestep so subsequent iters can resume v3.1-dev-netcdf-v2's write_restart only saves model state for the timesteps listed in output_settings$restart_save_timesteps (default: just forecast_start_datetime). The next iter computes start_datetime = prev_fsd - 4 days, which is not in the saved set, so generate_initial_ conditions can't find the right offset and aborts. 'all' tells write_restart to save every timestep; cheap (5 small snapshots in the zip) and removes the offset-coupling between the loop's start_datetime math and the restart save list. --- configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml index 0974556..8c0752c 100644 --- a/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml +++ b/configuration/glm_aed_flare_rs/configure_flare_glm_aed.yml @@ -100,6 +100,7 @@ output_settings: diagnostics_names: [extc] generate_plots: FALSE evaluate_past: FALSE + restart_save_timesteps: "all" diagnostics_daily: names: ['CAR_ch4_atm','CAR_atm_co2_flux', 'temp', 'OXY_oxy', 'temp', 'temp'] save_names: ['ch4_flux_mean', 'co2_flux_mean', 'temp_1.6m_mean', 'oxy_mean', 'temp_1.0m_mean', 'temp_8.0m_mean'] From 25a9fa4b2c51a8cccb45730803496f16941fe363 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 15 May 2026 13:49:24 -0700 Subject: [PATCH 39/44] Carry-forward start_datetime = old_fsd - 3 days for restart compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quinn's new write_restart on v3.1-dev-netcdf-v2 only persists GLM output timesteps in the zip — spinup_start (which is the date the team's '-4 days' pattern lands on) is never saved. generate_restart_initial_ conditions_from_zip then can't find that date and aborts with 'No GLM restart directory found for date X. Available dates: ...'. Shift to -3 days so the next iter's start_datetime lands on the earliest GLM-output date in the restart zip while still preserving a 4-day spinup window into the new forecast_start_datetime. --- workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index fee8c98..1f8dd89 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -310,7 +310,13 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config) forecast_start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) + lubridate::days(1) - start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) - lubridate::days(4) + # v3.1-dev-netcdf-v2's write_restart only persists GLM output timesteps + # (i.e. dates from spinup_start + 1 onward); the spinup_start date itself + # is never saved. The team's combined_run_aed.R pattern of `-4 days` + # would land on that un-saved start. Shifting to `-3 days` picks the + # earliest restart-available date while still giving the next iter a + # 4-day spinup window before its forecast_start. + start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) - lubridate::days(3) # Use the actual filename run_flare wrote, instead of reconstructing it. # write_restart returns .zip when GLM restart staging is enabled (Quinn's # v3.1-dev-netcdf-v2 default) and .nc otherwise. Hard-coding either From de978c1fc1f16d79a77877b0f6ba0a560b760077 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 20 May 2026 20:30:59 -0700 Subject: [PATCH 40/44] Tighten inline comments in run_fcre_aed_forecast --- .../glm_aed_flare_rs/run_fcre_aed_forecast.R | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R index 1f8dd89..026a947 100644 --- a/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R +++ b/workflows/glm_aed_flare_rs/run_fcre_aed_forecast.R @@ -5,9 +5,8 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", library(lubridate) set.seed(100) - # Resolve GLM_PATH: respect any pre-set value (container ENV or caller's - # Sys.setenv); else use the container's embedded binary if present; else - # fall back to GLM3r for local/S3 runs. + # Resolve GLM_PATH: respect any pre-set value, else use a container- + # embedded binary if available, else fall back to GLM3r. glm_path_initial <- Sys.getenv("GLM_PATH") message("GLM_PATH at function entry: '", glm_path_initial, "'") if (glm_path_initial == "" || glm_path_initial == "GLM3r") { @@ -44,11 +43,8 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", lake_directory = lake_directory, config_set_name = config_set_name) - # WORKAROUND: FLAREr::check_noaa_present catches arrow IOError when the - # NOAA partition is missing, but its handler does `message(error_message)` - # with the condition object, which re-signals the error and aborts the - # action instead of returning FALSE. Drop this wrapper once FLAREr's - # check_noaa_present uses message(conditionMessage(e)). + # Wrap check_noaa_present: its internal handler can re-signal an arrow + # IOError when the NOAA partition is missing, which would abort the run. noaa_ready <- tryCatch( FLAREr::check_noaa_present(lake_directory, configure_run_file, @@ -310,18 +306,12 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config) forecast_start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) + lubridate::days(1) - # v3.1-dev-netcdf-v2's write_restart only persists GLM output timesteps - # (i.e. dates from spinup_start + 1 onward); the spinup_start date itself - # is never saved. The team's combined_run_aed.R pattern of `-4 days` - # would land on that un-saved start. Shifting to `-3 days` picks the - # earliest restart-available date while still giving the next iter a - # 4-day spinup window before its forecast_start. + # write_restart persists GLM output timesteps starting from spinup_start + 1, + # so the next iteration's start_datetime must be at least one day after the + # current spinup_start to land on a saved restart date. start_datetime <- lubridate::as_datetime(config$run_config$forecast_start_datetime) - lubridate::days(3) - # Use the actual filename run_flare wrote, instead of reconstructing it. - # write_restart returns .zip when GLM restart staging is enabled (Quinn's - # v3.1-dev-netcdf-v2 default) and .nc otherwise. Hard-coding either - # extension would mismatch what was uploaded to S3, so get_restart_file - # would 404 on the next iteration. + # Read the actual filename produced by run_flare; the extension depends on + # whether GLM restart staging emitted a .zip or a plain .nc. restart_file <- basename(flare_result$restart_file) FLAREr::update_run_config(lake_directory = lake_directory, @@ -341,17 +331,9 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", config = config, use_https = TRUE) - # POSSIBLE BUG FIX -- safe to remove if upstream behavior changes. - # update_run_config just wrote a new forecast_start_datetime to - # disk + S3, but the in-memory `config` still holds the previous - # values. Sourced helpers on the next iteration - # (generate_inflow_forecast.R -> convert_vera4cast_inflow) read - # from this outer `config` and would write inflow partitions under - # the previous reference_date, while run_flare re-reads config - # from disk and looks under the new one -- producing an - # arrow::open_dataset IOError. Refreshing here keeps the in-memory - # view aligned with on-disk state. combined_run_aed.R doesn't need - # this because each cron firing runs the loop exactly once. + # Refresh the in-memory config so sourced helpers on the next iteration + # see the updated forecast_start_datetime / restart_file that + # update_run_config just wrote to disk. config <- FLAREr::set_up_simulation(configure_run_file = configure_run_file, lake_directory = lake_directory, config_set_name = config_set_name) @@ -367,7 +349,6 @@ run_fcre_aed_forecast <- function(config_set_name = "glm_aed_flare_rs", Sys.setenv("AWS_ACCESS_KEY_ID" = var1, "AWS_SECRET_ACCESS_KEY" = var2) - # See note above on the initial call site. noaa_ready <- tryCatch( FLAREr::check_noaa_present(lake_directory, configure_run_file, From 04a629ac55f6ee94e5b0294c05d850c47976e4ac Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 20 May 2026 20:30:59 -0700 Subject: [PATCH 41/44] Bump sim_name for fresh demo run --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 3efd68c..a01fad4 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -3,7 +3,7 @@ start_datetime: 2026-04-30 00:00:00 end_datetime: .na forecast_start_datetime: 2026-05-04 00:00:00 forecast_horizon: 1 -sim_name: glm_aed_flare_rs_demo_faasr +sim_name: glm_aed_flare_rs_demo_v2 configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml use_s3: TRUE From de9747b89c777ab03bb8a92fee0c13d515857c8e Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Wed, 20 May 2026 21:23:26 -0700 Subject: [PATCH 42/44] Bump sim_name to v3 for fresh demo iteration --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index a01fad4..6bc8ee7 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -3,7 +3,7 @@ start_datetime: 2026-04-30 00:00:00 end_datetime: .na forecast_start_datetime: 2026-05-04 00:00:00 forecast_horizon: 1 -sim_name: glm_aed_flare_rs_demo_v2 +sim_name: glm_aed_flare_rs_demo_v3 configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml use_s3: TRUE From 21fa9a589198fe5599c8c58a932f5dec327a7057 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 22 May 2026 08:52:02 -0700 Subject: [PATCH 43/44] Bump sim_name to v4 for fresh demo run --- configuration/glm_aed_flare_rs/configure_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/glm_aed_flare_rs/configure_run.yml b/configuration/glm_aed_flare_rs/configure_run.yml index 6bc8ee7..be8c9cd 100644 --- a/configuration/glm_aed_flare_rs/configure_run.yml +++ b/configuration/glm_aed_flare_rs/configure_run.yml @@ -3,7 +3,7 @@ start_datetime: 2026-04-30 00:00:00 end_datetime: .na forecast_start_datetime: 2026-05-04 00:00:00 forecast_horizon: 1 -sim_name: glm_aed_flare_rs_demo_v3 +sim_name: glm_aed_flare_rs_demo_v4 configure_flare: configure_flare_glm_aed.yml configure_obs: observation_processing.yml use_s3: TRUE From da4f0dd316e2809bab8505f37e7eb7d4bcfdd147 Mon Sep 17 00:00:00 2001 From: Ashish-Ramrakhiani Date: Fri, 22 May 2026 15:15:19 -0700 Subject: [PATCH 44/44] Restore keep_automation.txt to preserve upstream automation-keepalive heartbeat --- keep_automation.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/keep_automation.txt b/keep_automation.txt index 87f50c2..c9d4c9e 100644 --- a/keep_automation.txt +++ b/keep_automation.txt @@ -26,3 +26,4 @@ Thu Jan 1 01:11:26 AM UTC 2026 Sun Feb 1 01:28:02 AM UTC 2026 Sun Mar 1 01:26:18 AM UTC 2026 Wed Apr 1 01:50:03 AM UTC 2026 +Fri May 1 02:11:09 AM UTC 2026