From 83bff2401975057d8ec05d1f263b01f3c88668fe Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Thu, 11 Sep 2025 10:38:38 -0400 Subject: [PATCH 01/19] feat: config environment --- R/config.R | 1 + 1 file changed, 1 insertion(+) create mode 100644 R/config.R diff --git a/R/config.R b/R/config.R new file mode 100644 index 0000000..1654194 --- /dev/null +++ b/R/config.R @@ -0,0 +1 @@ +config <- new.env() \ No newline at end of file From 54717b3020687156278815f56a5cc1809e6a2cd3 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Thu, 11 Sep 2025 10:39:21 -0400 Subject: [PATCH 02/19] feat: config-dependent variables moved to config env --- R/globals.R | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/R/globals.R b/R/globals.R index d911a16..0ac84da 100644 --- a/R/globals.R +++ b/R/globals.R @@ -1,18 +1,17 @@ # Define extent of exported data (ai_app_extent) -ai_app_crs <- sf::st_crs("EPSG:3857") +config$ai_app_crs <- sf::st_crs("EPSG:3857") # Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -ai_app_extent <- c(-18924313.4348565, -5565974.53966368, +config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) # Configure S3 bucket, its url and the path within it used for flow output -s3_bucket_name <- "avianinfluenza" -s3_flow_path <- "flow/" -s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" - +config$s3_bucket_name <- "avianinfluenza" +config$s3_flow_path <- "flow/" +config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" # Define local cache for temporary output images # Will then be copied to AWS -local_cache <- tempdir() +config$local_cache <- tempdir() if(!file.exists(local_cache)) dir.create(local_cache) From d7aa22a6b13153dcb70c610a1ee317580fbe6194 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Thu, 11 Sep 2025 10:40:01 -0400 Subject: [PATCH 03/19] feat: move state variables created by set_s3_config and retrieved by get_s3_config to config environment --- R/flow.R | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/R/flow.R b/R/flow.R index 2b81f7b..fddd4ac 100644 --- a/R/flow.R +++ b/R/flow.R @@ -2,26 +2,27 @@ #' #' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { - assign(".s3_config", list( + config$option <- list( access_key = access_key, secret_key = secret_key, region = region, bucket = bucket - ), envir = .GlobalEnv) + ) } #' Get S3 configuration from explicit config, environment variables, or IAM role get_s3_config <- function() { - config <- if (exists(".s3_config", envir = .GlobalEnv)) get(".s3_config", envir = .GlobalEnv) else list() - access_key <- config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) - secret_key <- config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) - region <- config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) - bucket <- config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) + config_option_list <- config$option %||% list() + access_key <- config_option_list$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) + secret_key <- config_option_list$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) + region <- config_option_list$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) + bucket <- config_option_list$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) } #' Null coalescing operator for config values `%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b + #' @import BirdFlowR if(FALSE) { From 0ab83e746073d50ad113751455a1b736da2c0d6d Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:07:07 -0400 Subject: [PATCH 04/19] fix: delete old config file --- R/config.R | 1 - 1 file changed, 1 deletion(-) delete mode 100644 R/config.R diff --git a/R/config.R b/R/config.R deleted file mode 100644 index 1654194..0000000 --- a/R/config.R +++ /dev/null @@ -1 +0,0 @@ -config <- new.env() \ No newline at end of file From 23a95c33ded07596d48dfcc635e87968b45f6b4f Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:08:31 -0400 Subject: [PATCH 05/19] feat: new s3_config file with s3_config environment definition and get, set, helper functions --- R/flow.R | 25 ------------------------- R/s3_config.R | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) create mode 100644 R/s3_config.R diff --git a/R/flow.R b/R/flow.R index cde7be1..3231c02 100644 --- a/R/flow.R +++ b/R/flow.R @@ -1,28 +1,3 @@ -#' Set S3 configuration explicitly -#' -#' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. -set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { - config$option <- list( - access_key = access_key, - secret_key = secret_key, - region = region, - bucket = bucket - ) -} - -#' Get S3 configuration from explicit config, environment variables, or IAM role -get_s3_config <- function() { - config_option_list <- config$option %||% list() - access_key <- config_option_list$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) - secret_key <- config_option_list$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) - region <- config_option_list$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) - bucket <- config_option_list$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) - list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) -} - -#' Null coalescing operator for config values -`%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b - if(FALSE) { # Manually set function arguments for dev and debugging # Running code here allows running function body code outside of the diff --git a/R/s3_config.R b/R/s3_config.R new file mode 100644 index 0000000..9aa32ee --- /dev/null +++ b/R/s3_config.R @@ -0,0 +1,26 @@ +s3_config <- new.env() + +#' Set S3 configuration explicitly +#' +#' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. +set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { + assign(".s3_config", list( + access_key = access_key, + secret_key = secret_key, + region = region, + bucket = bucket + ), envir = .GlobalEnv) +} + +#' Get S3 configuration from explicit config, environment variables, or IAM role +get_s3_config <- function() { + config <- if (exists(".s3_config", envir = .GlobalEnv)) get(".s3_config", envir = .GlobalEnv) else list() + access_key <- config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) + secret_key <- config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) + region <- config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) + bucket <- config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) + list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) +} + +#' Null coalescing operator for config values +`%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b \ No newline at end of file From 0c076f57a1eeabd0dda6a2d485a64c04414ece60 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:10:26 -0400 Subject: [PATCH 06/19] fix: rename config$ to s3_config$ --- R/globals.R | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/R/globals.R b/R/globals.R index 0ac84da..8dbaf94 100644 --- a/R/globals.R +++ b/R/globals.R @@ -1,17 +1,17 @@ # Define extent of exported data (ai_app_extent) -config$ai_app_crs <- sf::st_crs("EPSG:3857") +s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") # Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, +s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) -# Configure S3 bucket, its url and the path within it used for flow output -config$s3_bucket_name <- "avianinfluenza" -config$s3_flow_path <- "flow/" -config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" +# s3_configure S3 bucket, its url and the path within it used for flow output +s3_config$s3_bucket_name <- "avianinfluenza" +s3_config$s3_flow_path <- "flow/" +s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" # Define local cache for temporary output images # Will then be copied to AWS -config$local_cache <- tempdir() +s3_config$local_cache <- tempdir() if(!file.exists(local_cache)) - dir.create(local_cache) + dir.create(s3_config$local_cache) From af15577fa9230047d18efe6d2bbfc87eca0d4748 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:14:01 -0400 Subject: [PATCH 07/19] feat: s3_config env in get and set config functions --- R/s3_config.R | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/R/s3_config.R b/R/s3_config.R index 9aa32ee..9386d5d 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -4,23 +4,20 @@ s3_config <- new.env() #' #' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { - assign(".s3_config", list( - access_key = access_key, - secret_key = secret_key, - region = region, - bucket = bucket - ), envir = .GlobalEnv) + s3_config$access_key <- access_key + s3_config$secret_key <- secret_key + s3_config$region <- region + s3_config$bucket <- bucket } #' Get S3 configuration from explicit config, environment variables, or IAM role get_s3_config <- function() { - config <- if (exists(".s3_config", envir = .GlobalEnv)) get(".s3_config", envir = .GlobalEnv) else list() - access_key <- config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) - secret_key <- config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) - region <- config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) - bucket <- config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) + access_key <- s3_config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) + secret_key <- s3_config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) + region <- s3_config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) + bucket <- s3_config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) } #' Null coalescing operator for config values -`%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b \ No newline at end of file +`%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b From 56a00481746b168b06efbe91885109e5d5e182f8 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:16:01 -0400 Subject: [PATCH 08/19] feat: move global config vars to s3_config.R --- R/globals.R | 17 ----------------- R/s3_config.R | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) delete mode 100644 R/globals.R diff --git a/R/globals.R b/R/globals.R deleted file mode 100644 index 8dbaf94..0000000 --- a/R/globals.R +++ /dev/null @@ -1,17 +0,0 @@ -# Define extent of exported data (ai_app_extent) -s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") - -# Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, - 1118889.97485796, 15538711.0963092) - -# s3_configure S3 bucket, its url and the path within it used for flow output -s3_config$s3_bucket_name <- "avianinfluenza" -s3_config$s3_flow_path <- "flow/" -s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" - -# Define local cache for temporary output images -# Will then be copied to AWS -s3_config$local_cache <- tempdir() -if(!file.exists(local_cache)) - dir.create(s3_config$local_cache) diff --git a/R/s3_config.R b/R/s3_config.R index 9386d5d..6496bb5 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -1,5 +1,22 @@ s3_config <- new.env() +# Define extent of exported data (ai_app_extent) +s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") + +# Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 +s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, + 1118889.97485796, 15538711.0963092) + +# s3_configure S3 bucket, its url and the path within it used for flow output +s3_config$s3_bucket_name <- "avianinfluenza" +s3_config$s3_flow_path <- "flow/" +s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" + +# Define local cache for temporary output images, will then be copied to AWS +s3_config$local_cache <- tempdir() +if(!file.exists(s3_config$local_cache)) + dir.create(s3_config$local_cache) + #' Set S3 configuration explicitly #' #' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. From 0aa02fa6178be34484a8a577f49cb0abc35153e2 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:16:55 -0400 Subject: [PATCH 09/19] chore: remove helper comments and documentation --- R/s3_config.R | 8 +------- man/get_s3_config.Rd | 11 ----------- man/grapes-or-or-grapes.Rd | 11 ----------- man/set_s3_config.Rd | 16 ---------------- 4 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 man/get_s3_config.Rd delete mode 100644 man/grapes-or-or-grapes.Rd delete mode 100644 man/set_s3_config.Rd diff --git a/R/s3_config.R b/R/s3_config.R index 6496bb5..4a14a03 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -4,8 +4,7 @@ s3_config <- new.env() s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") # Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, - 1118889.97485796, 15538711.0963092) +s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) # s3_configure S3 bucket, its url and the path within it used for flow output s3_config$s3_bucket_name <- "avianinfluenza" @@ -17,9 +16,6 @@ s3_config$local_cache <- tempdir() if(!file.exists(s3_config$local_cache)) dir.create(s3_config$local_cache) -#' Set S3 configuration explicitly -#' -#' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { s3_config$access_key <- access_key s3_config$secret_key <- secret_key @@ -27,7 +23,6 @@ set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, b s3_config$bucket <- bucket } -#' Get S3 configuration from explicit config, environment variables, or IAM role get_s3_config <- function() { access_key <- s3_config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) secret_key <- s3_config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) @@ -36,5 +31,4 @@ get_s3_config <- function() { list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) } -#' Null coalescing operator for config values `%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b diff --git a/man/get_s3_config.Rd b/man/get_s3_config.Rd deleted file mode 100644 index 71b2e21..0000000 --- a/man/get_s3_config.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/flow.R -\name{get_s3_config} -\alias{get_s3_config} -\title{Get S3 configuration from explicit config, environment variables, or IAM role} -\usage{ -get_s3_config() -} -\description{ -Get S3 configuration from explicit config, environment variables, or IAM role -} diff --git a/man/grapes-or-or-grapes.Rd b/man/grapes-or-or-grapes.Rd deleted file mode 100644 index 6c41ad5..0000000 --- a/man/grapes-or-or-grapes.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/flow.R -\name{\%||\%} -\alias{\%||\%} -\title{Null coalescing operator for config values} -\usage{ -a \%||\% b -} -\description{ -Null coalescing operator for config values -} diff --git a/man/set_s3_config.Rd b/man/set_s3_config.Rd deleted file mode 100644 index c97a5e1..0000000 --- a/man/set_s3_config.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/flow.R -\name{set_s3_config} -\alias{set_s3_config} -\title{Set S3 configuration explicitly} -\usage{ -set_s3_config( - access_key = NULL, - secret_key = NULL, - region = NULL, - bucket = NULL -) -} -\description{ -Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. -} From 07338a98c8c3e4f3ecde886ddc76a692fa2b77c2 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 19 Sep 2025 16:29:06 -0400 Subject: [PATCH 10/19] feat: s3_config$ prefix for flow variables --- R/flow.R | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/R/flow.R b/R/flow.R index 3231c02..35ac373 100644 --- a/R/flow.R +++ b/R/flow.R @@ -132,11 +132,11 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) pred_weeks <- BirdFlowR::lookup_timestep_sequence(bf, start = week, n = n, direction = direction) png_files <- paste0(flow_type, "_", taxa, "_", pred_weeks, ".png") symbology_files <- paste0(flow_type, "_", taxa, "_", pred_weeks, ".json") - png_bucket_paths <- paste0(s3_flow_path, cache_prefix, png_files) - symbology_bucket_paths <- paste0(s3_flow_path, cache_prefix, symbology_files) - png_urls <- paste0(s3_flow_url, cache_prefix, png_files) - symbology_urls <- paste0(s3_flow_url, cache_prefix, symbology_files) - tiff_bucket_path <- paste0(s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") + png_bucket_paths <- paste0(s3_config$s3_flow_path, cache_prefix, png_files) + symbology_bucket_paths <- paste0(s3_config$s3_flow_path, cache_prefix, symbology_files) + png_urls <- paste0(s3_config$s3_flow_url, cache_prefix, png_files) + symbology_urls <- paste0(s3_config$s3_flow_url, cache_prefix, symbology_files) + tiff_bucket_path <- paste0(s3_config$s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") # --- CACHE CHECK BLOCK --- cache_hit <- TRUE @@ -179,7 +179,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) start = list(week = week, taxa = taxa, loc = loc), status = "cached", result = result, - geotiff = if (save_local) tiff_local_path else paste0(s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") + geotiff = if (save_local) tiff_local_path else paste0(s3_config$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") ) ) } @@ -240,25 +240,25 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) log_progress("Projecting and cropping raster for web output.") log_progress(paste0("combined class: ", class(combined))) - log_progress(paste0("ai_app_crs$input: ", ai_app_crs$input)) - log_progress(paste0("ai_app_extent: ", ai_app_extent)) + log_progress(paste0("s3_config$ai_app_crs$input: ", s3_config$ai_app_crs$input)) + log_progress(paste0("s3_config$ai_app_extent: ", s3_config$ai_app_extent)) if (is.null(combined) || !inherits(combined, "SpatRaster")) { log_progress("ERROR: combined raster is NULL or not a SpatRaster. Aborting.") return(format_error("combined raster is NULL or not a SpatRaster")) } - if (is.null(ai_app_crs$input)) { - log_progress("ERROR: ai_app_crs$input is NULL. Aborting.") - return(format_error("ai_app_crs$input is NULL")) + if (is.null(s3_config$ai_app_crs$input)) { + log_progress("ERROR: s3_config$ai_app_crs$input is NULL. Aborting.") + return(format_error("s3_config$ai_app_crs$input is NULL")) } - if (is.null(ai_app_extent)) { - log_progress("ERROR: ai_app_extent is NULL. Aborting.") - return(format_error("ai_app_extent is NULL")) + if (is.null(s3_config$ai_app_extent)) { + log_progress("ERROR: s3_config$ai_app_extent is NULL. Aborting.") + return(format_error("s3_config$ai_app_extent is NULL")) } log_progress("Projecting") - web_raster <- combined |> terra::project(ai_app_crs$input) + web_raster <- combined |> terra::project(s3_config$ai_app_crs$input) log_progress("Cropping") - web_raster <- terra::crop(web_raster, ai_app_extent) + web_raster <- terra::crop(web_raster, s3_config$ai_app_extent) log_progress("Done cropping") png_paths <- file.path(out_path, png_files) symbology_paths <- file.path(out_path, symbology_files) @@ -346,7 +346,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) start = list(week = week, taxa = taxa, loc = loc), status = "success", result = result, - geotiff = if (save_local) tiff_path else paste0(s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") + geotiff = if (save_local) tiff_path else paste0(s3_config$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") ) ) } From 6aec47a8105888eda5df25365289619d4caa95b6 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 16:28:11 -0400 Subject: [PATCH 11/19] feat: move all s3_config params to set_s3_config --- R/s3_config.R | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/R/s3_config.R b/R/s3_config.R index 4a14a03..6886a59 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -1,26 +1,34 @@ s3_config <- new.env() # Define extent of exported data (ai_app_extent) -s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") +# s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") # Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) +# s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) # s3_configure S3 bucket, its url and the path within it used for flow output -s3_config$s3_bucket_name <- "avianinfluenza" -s3_config$s3_flow_path <- "flow/" -s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" +# s3_config$s3_bucket_name <- "avianinfluenza" +# s3_config$s3_flow_path <- "flow/" +# s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" # Define local cache for temporary output images, will then be copied to AWS -s3_config$local_cache <- tempdir() -if(!file.exists(s3_config$local_cache)) - dir.create(s3_config$local_cache) +# s3_config$local_cache <- tempdir() +# if(!file.exists(s3_config$local_cache)) +# dir.create(s3_config$local_cache) set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { s3_config$access_key <- access_key s3_config$secret_key <- secret_key s3_config$region <- region s3_config$bucket <- bucket + s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") + s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) + s3_config$s3_bucket_name <- "avianinfluenza" + s3_config$s3_flow_path <- "flow/" + s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" + s3_config$local_cache <- tempdir() + if(!file.exists(s3_config$local_cache)) + dir.create(s3_config$local_cache) } get_s3_config <- function() { From dfde6cbbb0d7f6a853d10ab10edef5e8e5ee7dfb Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 16:29:12 -0400 Subject: [PATCH 12/19] style: potential TODO item --- R/s3_config.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/s3_config.R b/R/s3_config.R index 6886a59..001452e 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -31,6 +31,7 @@ set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, b dir.create(s3_config$local_cache) } +# TODO: add all s3_config params here? get_s3_config <- function() { access_key <- s3_config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) secret_key <- s3_config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) From 27ec4458edcdd4dd50ed1a6d42030454c75fe211 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 16:33:34 -0400 Subject: [PATCH 13/19] feat: set s3_config before tests --- tests/testthat/setup.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index d6b67ae..f4be360 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1 +1,2 @@ -load_models() \ No newline at end of file +load_models() +set_s3_config() From a674d2b0f500c5e7f682a0ddf5c75ef739124d5c Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 16:42:11 -0400 Subject: [PATCH 14/19] feat: log flag to turn logging on/off --- R/flow.R | 4 +++- R/s3_config.R | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/R/flow.R b/R/flow.R index 35ac373..42045fe 100644 --- a/R/flow.R +++ b/R/flow.R @@ -85,7 +85,9 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) } log_progress <- function(msg) { - cat(sprintf("[%s] %s\n", Sys.time(), msg), file = "./flow_debug.log", append = TRUE) + if(s3_config$log) { + cat(sprintf("[%s] %s\n", Sys.time(), msg), file = "./flow_debug.log", append = TRUE) + } } log_progress(paste0("Starting flow function with arguments: loc=", loc, ", week=", week, ", taxa=", taxa, ", n=", n, ", direction=", direction, ", save_local=", save_local)) diff --git a/R/s3_config.R b/R/s3_config.R index 001452e..184dbb6 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -16,11 +16,12 @@ s3_config <- new.env() # if(!file.exists(s3_config$local_cache)) # dir.create(s3_config$local_cache) -set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { +set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE) { s3_config$access_key <- access_key s3_config$secret_key <- secret_key s3_config$region <- region s3_config$bucket <- bucket + s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) s3_config$s3_bucket_name <- "avianinfluenza" @@ -29,6 +30,8 @@ set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, b s3_config$local_cache <- tempdir() if(!file.exists(s3_config$local_cache)) dir.create(s3_config$local_cache) + + s3_config$log <- log } # TODO: add all s3_config params here? From acaa61e0f4be2f910f3d442a66043a36435a0176 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 16:45:45 -0400 Subject: [PATCH 15/19] feat: log_file_path parameter for flow to store logs --- R/flow.R | 2 +- R/s3_config.R | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/R/flow.R b/R/flow.R index 42045fe..0bcfb9b 100644 --- a/R/flow.R +++ b/R/flow.R @@ -86,7 +86,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) log_progress <- function(msg) { if(s3_config$log) { - cat(sprintf("[%s] %s\n", Sys.time(), msg), file = "./flow_debug.log", append = TRUE) + cat(sprintf("[%s] %s\n", Sys.time(), msg), file = s3_config$log_file_path, append = TRUE) } } diff --git a/R/s3_config.R b/R/s3_config.R index 184dbb6..f396f00 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -16,7 +16,7 @@ s3_config <- new.env() # if(!file.exists(s3_config$local_cache)) # dir.create(s3_config$local_cache) -set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE) { +set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE, log_file_path = "./flow_debug.log") { s3_config$access_key <- access_key s3_config$secret_key <- secret_key s3_config$region <- region @@ -32,6 +32,7 @@ set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, b dir.create(s3_config$local_cache) s3_config$log <- log + s3_config$log_file_path <- log_file_path } # TODO: add all s3_config params here? From 2878bcf977c0bc04478544364967826f1fae02c6 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 16:57:46 -0400 Subject: [PATCH 16/19] feat: local_temp_path variable for writing files locally --- R/flow.R | 13 +++++++------ R/s3_config.R | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/R/flow.R b/R/flow.R index 0bcfb9b..a4bc820 100644 --- a/R/flow.R +++ b/R/flow.R @@ -139,6 +139,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) png_urls <- paste0(s3_config$s3_flow_url, cache_prefix, png_files) symbology_urls <- paste0(s3_config$s3_flow_url, cache_prefix, symbology_files) tiff_bucket_path <- paste0(s3_config$s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") + local_temp_path <- s3_config$local_temp_path # --- CACHE CHECK BLOCK --- cache_hit <- TRUE @@ -154,9 +155,9 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) tiff_exists <- aws.s3::object_exists(object = tiff_bucket_path, bucket = s3_cfg$bucket) if (!tiff_exists) cache_hit <- FALSE } else { - # Local cache: check if all files exist in localtmp - dir.create("localtmp", showWarnings = FALSE) - local_cache_prefix <- file.path("localtmp", gsub("/", "_", cache_prefix)) + # Local cache: check if all files exist in local_temp_path + dir.create(local_temp_path, showWarnings = FALSE) + local_cache_prefix <- file.path(local_temp_path, gsub("/", "_", cache_prefix)) png_local_paths <- file.path(local_cache_prefix, png_files) json_local_paths <- file.path(local_cache_prefix, symbology_files) tiff_local_path <- file.path(local_cache_prefix, paste0(flow_type, "_", taxa, ".tif")) @@ -175,7 +176,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) ) log_progress(paste0("Cached result for week ", pred_weeks[i], ": url=", result[[i]]$url, ", legend=", result[[i]]$legend)) } - log_progress(if (save_local) "Returned cached result from localtmp" else "Returned cached result from S3") + log_progress(if (save_local) "Returned cached result from local_temp_path" else "Returned cached result from S3") return( list( start = list(week = week, taxa = taxa, loc = loc), @@ -189,8 +190,8 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) # Continue with prediction if (save_local || !s3_enabled) { - dir.create("localtmp", showWarnings = FALSE) - out_path <- file.path("localtmp", gsub("/", "_", cache_prefix)) + dir.create(local_temp_path, showWarnings = FALSE) + out_path <- file.path(local_temp_path, gsub("/", "_", cache_prefix)) dir.create(out_path, recursive = TRUE, showWarnings = FALSE) } else { out_path <- tempfile(pattern = "flow_", tmpdir = "/dev/shm") diff --git a/R/s3_config.R b/R/s3_config.R index f396f00..c6c788a 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -16,7 +16,7 @@ s3_config <- new.env() # if(!file.exists(s3_config$local_cache)) # dir.create(s3_config$local_cache) -set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE, log_file_path = "./flow_debug.log") { +set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE, log_file_path = "./flow_debug.log", local_temp_path = "localtmp") { s3_config$access_key <- access_key s3_config$secret_key <- secret_key s3_config$region <- region @@ -33,6 +33,7 @@ set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, b s3_config$log <- log s3_config$log_file_path <- log_file_path + s3_config$local_temp_path <- local_temp_path } # TODO: add all s3_config params here? From a49d11c850199e4f55e3b14f5713f914b4345914 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Wed, 24 Sep 2025 17:23:30 -0400 Subject: [PATCH 17/19] docs: add sf to imports --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 896f281..900ea82 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -13,7 +13,8 @@ Imports: plumber, jsonlite, aws.s3, - terra + terra, + sf Suggests: testthat (>= 3.0.0), callthat, From 36e16381e6d14cc88106d5899722a0ba8897bb76 Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 26 Sep 2025 16:34:00 -0400 Subject: [PATCH 18/19] feat: add all s3_config variables to get_s3_config and generalize null coalescing function --- R/s3_config.R | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/R/s3_config.R b/R/s3_config.R index c6c788a..16ad164 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -36,13 +36,29 @@ set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, b s3_config$local_temp_path <- local_temp_path } -# TODO: add all s3_config params here? get_s3_config <- function() { - access_key <- s3_config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) - secret_key <- s3_config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) - region <- s3_config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) - bucket <- s3_config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) - list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) + list( + access_key = s3_config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA), + secret_key = s3_config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA), + region = s3_config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA), + bucket = s3_config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA), + ai_app_crs = s3_config$ai_app_crs %||% NA, + ai_app_extent = s3_config$ai_app_extent %||% NA, + s3_bucket_name = s3_config$s3_bucket_name %||% NA, + s3_flow_path = s3_config$s3_flow_path %||% NA, + s3_flow_url = s3_config$s3_flow_url %||% NA, + local_cache = s3_config$local_cache %||% NA, + log = s3_config$log %||% NA, + log_file_path = s3_config$log_file_path %||% NA, + local_temp_path = s3_config$local_temp_path %||% NA + ) } -`%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b +# `%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b + +`%||%` <- function(a, b) { + if (is.null(a)) return(b) + if (is.atomic(a) && length(a) == 1 && is.na(a)) return(b) + if (is.character(a) && length(a) == 1 && !nzchar(a)) return(b) + a +} From aa9d72226a1db21f1e68ce663a80d1c6987013da Mon Sep 17 00:00:00 2001 From: Atharva Shahane Date: Fri, 26 Sep 2025 16:37:29 -0400 Subject: [PATCH 19/19] feat: use get_s3_config in flow --- R/flow.R | 45 ++++++++++++++++++++------------------------- R/s3_config.R | 18 ------------------ 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/R/flow.R b/R/flow.R index a4bc820..c7d7351 100644 --- a/R/flow.R +++ b/R/flow.R @@ -67,11 +67,6 @@ save_local_path <- "config/save_local.flag" #' `type` #' @export flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) { - # TODO: - # utils::data("species", package = "BirdFlowAPI", envir = environment()) - - # load_models() - s3_cfg <- get_s3_config() s3_enabled <- !is.na(s3_cfg$bucket) && nzchar(s3_cfg$bucket) @@ -85,8 +80,8 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) } log_progress <- function(msg) { - if(s3_config$log) { - cat(sprintf("[%s] %s\n", Sys.time(), msg), file = s3_config$log_file_path, append = TRUE) + if(s3_cfg$log) { + cat(sprintf("[%s] %s\n", Sys.time(), msg), file = s3_cfg$log_file_path, append = TRUE) } } @@ -134,12 +129,12 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) pred_weeks <- BirdFlowR::lookup_timestep_sequence(bf, start = week, n = n, direction = direction) png_files <- paste0(flow_type, "_", taxa, "_", pred_weeks, ".png") symbology_files <- paste0(flow_type, "_", taxa, "_", pred_weeks, ".json") - png_bucket_paths <- paste0(s3_config$s3_flow_path, cache_prefix, png_files) - symbology_bucket_paths <- paste0(s3_config$s3_flow_path, cache_prefix, symbology_files) - png_urls <- paste0(s3_config$s3_flow_url, cache_prefix, png_files) - symbology_urls <- paste0(s3_config$s3_flow_url, cache_prefix, symbology_files) - tiff_bucket_path <- paste0(s3_config$s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") - local_temp_path <- s3_config$local_temp_path + png_bucket_paths <- paste0(s3_cfg$s3_flow_path, cache_prefix, png_files) + symbology_bucket_paths <- paste0(s3_cfg$s3_flow_path, cache_prefix, symbology_files) + png_urls <- paste0(s3_cfg$s3_flow_url, cache_prefix, png_files) + symbology_urls <- paste0(s3_cfg$s3_flow_url, cache_prefix, symbology_files) + tiff_bucket_path <- paste0(s3_cfg$s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") + local_temp_path <- s3_cfg$local_temp_path # --- CACHE CHECK BLOCK --- cache_hit <- TRUE @@ -182,7 +177,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) start = list(week = week, taxa = taxa, loc = loc), status = "cached", result = result, - geotiff = if (save_local) tiff_local_path else paste0(s3_config$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") + geotiff = if (save_local) tiff_local_path else paste0(s3_cfg$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") ) ) } @@ -243,25 +238,25 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) log_progress("Projecting and cropping raster for web output.") log_progress(paste0("combined class: ", class(combined))) - log_progress(paste0("s3_config$ai_app_crs$input: ", s3_config$ai_app_crs$input)) - log_progress(paste0("s3_config$ai_app_extent: ", s3_config$ai_app_extent)) + log_progress(paste0("s3_cfg$ai_app_crs$input: ", s3_cfg$ai_app_crs$input)) + log_progress(paste0("s3_cfg$ai_app_extent: ", s3_cfg$ai_app_extent)) if (is.null(combined) || !inherits(combined, "SpatRaster")) { log_progress("ERROR: combined raster is NULL or not a SpatRaster. Aborting.") return(format_error("combined raster is NULL or not a SpatRaster")) } - if (is.null(s3_config$ai_app_crs$input)) { - log_progress("ERROR: s3_config$ai_app_crs$input is NULL. Aborting.") - return(format_error("s3_config$ai_app_crs$input is NULL")) + if (is.null(s3_cfg$ai_app_crs$input)) { + log_progress("ERROR: s3_cfg$ai_app_crs$input is NULL. Aborting.") + return(format_error("s3_cfg$ai_app_crs$input is NULL")) } - if (is.null(s3_config$ai_app_extent)) { - log_progress("ERROR: s3_config$ai_app_extent is NULL. Aborting.") - return(format_error("s3_config$ai_app_extent is NULL")) + if (is.null(s3_cfg$ai_app_extent)) { + log_progress("ERROR: s3_cfg$ai_app_extent is NULL. Aborting.") + return(format_error("s3_cfg$ai_app_extent is NULL")) } log_progress("Projecting") - web_raster <- combined |> terra::project(s3_config$ai_app_crs$input) + web_raster <- combined |> terra::project(s3_cfg$ai_app_crs$input) log_progress("Cropping") - web_raster <- terra::crop(web_raster, s3_config$ai_app_extent) + web_raster <- terra::crop(web_raster, s3_cfg$ai_app_extent) log_progress("Done cropping") png_paths <- file.path(out_path, png_files) symbology_paths <- file.path(out_path, symbology_files) @@ -349,7 +344,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) start = list(week = week, taxa = taxa, loc = loc), status = "success", result = result, - geotiff = if (save_local) tiff_path else paste0(s3_config$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") + geotiff = if (save_local) tiff_path else paste0(s3_cfg$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") ) ) } diff --git a/R/s3_config.R b/R/s3_config.R index 16ad164..4c88da8 100644 --- a/R/s3_config.R +++ b/R/s3_config.R @@ -1,21 +1,5 @@ s3_config <- new.env() -# Define extent of exported data (ai_app_extent) -# s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") - -# Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -# s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) - -# s3_configure S3 bucket, its url and the path within it used for flow output -# s3_config$s3_bucket_name <- "avianinfluenza" -# s3_config$s3_flow_path <- "flow/" -# s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" - -# Define local cache for temporary output images, will then be copied to AWS -# s3_config$local_cache <- tempdir() -# if(!file.exists(s3_config$local_cache)) -# dir.create(s3_config$local_cache) - set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE, log_file_path = "./flow_debug.log", local_temp_path = "localtmp") { s3_config$access_key <- access_key s3_config$secret_key <- secret_key @@ -54,8 +38,6 @@ get_s3_config <- function() { ) } -# `%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b - `%||%` <- function(a, b) { if (is.null(a)) return(b) if (is.atomic(a) && length(a) == 1 && is.na(a)) return(b)