diff --git a/DESCRIPTION b/DESCRIPTION index 896f281..900ea82 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -13,7 +13,8 @@ Imports: plumber, jsonlite, aws.s3, - terra + terra, + sf Suggests: testthat (>= 3.0.0), callthat, diff --git a/R/flow.R b/R/flow.R index 3a339ee..c7d7351 100644 --- a/R/flow.R +++ b/R/flow.R @@ -1,28 +1,3 @@ -#' Set S3 configuration explicitly -#' -#' Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. -set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL) { - assign(".s3_config", list( - access_key = access_key, - secret_key = secret_key, - region = region, - bucket = bucket - ), envir = .GlobalEnv) -} - -#' Get S3 configuration from explicit config, environment variables, or IAM role -get_s3_config <- function() { - config <- if (exists(".s3_config", envir = .GlobalEnv)) get(".s3_config", envir = .GlobalEnv) else list() - access_key <- config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA) - secret_key <- config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA) - region <- config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA) - bucket <- config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA) - list(access_key = access_key, secret_key = secret_key, region = region, bucket = bucket) -} - -#' Null coalescing operator for config values -`%||%` <- function(a, b) if (!is.null(a) && !is.na(a) && nzchar(a)) a else b - if(FALSE) { # Manually set function arguments for dev and debugging # Running code here allows running function body code outside of the @@ -92,11 +67,6 @@ save_local_path <- "config/save_local.flag" #' `type` #' @export flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) { - # TODO: - # utils::data("species", package = "BirdFlowAPI", envir = environment()) - - # load_models() - s3_cfg <- get_s3_config() s3_enabled <- !is.na(s3_cfg$bucket) && nzchar(s3_cfg$bucket) @@ -110,7 +80,9 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) } log_progress <- function(msg) { - cat(sprintf("[%s] %s\n", Sys.time(), msg), file = "./flow_debug.log", append = TRUE) + if(s3_cfg$log) { + cat(sprintf("[%s] %s\n", Sys.time(), msg), file = s3_cfg$log_file_path, append = TRUE) + } } log_progress(paste0("Starting flow function with arguments: loc=", loc, ", week=", week, ", taxa=", taxa, ", n=", n, ", direction=", direction, ", save_local=", save_local)) @@ -157,11 +129,12 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) pred_weeks <- BirdFlowR::lookup_timestep_sequence(bf, start = week, n = n, direction = direction) png_files <- paste0(flow_type, "_", taxa, "_", pred_weeks, ".png") symbology_files <- paste0(flow_type, "_", taxa, "_", pred_weeks, ".json") - png_bucket_paths <- paste0(s3_flow_path, cache_prefix, png_files) - symbology_bucket_paths <- paste0(s3_flow_path, cache_prefix, symbology_files) - png_urls <- paste0(s3_flow_url, cache_prefix, png_files) - symbology_urls <- paste0(s3_flow_url, cache_prefix, symbology_files) - tiff_bucket_path <- paste0(s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") + png_bucket_paths <- paste0(s3_cfg$s3_flow_path, cache_prefix, png_files) + symbology_bucket_paths <- paste0(s3_cfg$s3_flow_path, cache_prefix, symbology_files) + png_urls <- paste0(s3_cfg$s3_flow_url, cache_prefix, png_files) + symbology_urls <- paste0(s3_cfg$s3_flow_url, cache_prefix, symbology_files) + tiff_bucket_path <- paste0(s3_cfg$s3_flow_path, cache_prefix, flow_type, "_", taxa, ".tif") + local_temp_path <- s3_cfg$local_temp_path # --- CACHE CHECK BLOCK --- cache_hit <- TRUE @@ -177,9 +150,9 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) tiff_exists <- aws.s3::object_exists(object = tiff_bucket_path, bucket = s3_cfg$bucket) if (!tiff_exists) cache_hit <- FALSE } else { - # Local cache: check if all files exist in localtmp - dir.create("localtmp", showWarnings = FALSE) - local_cache_prefix <- file.path("localtmp", gsub("/", "_", cache_prefix)) + # Local cache: check if all files exist in local_temp_path + dir.create(local_temp_path, showWarnings = FALSE) + local_cache_prefix <- file.path(local_temp_path, gsub("/", "_", cache_prefix)) png_local_paths <- file.path(local_cache_prefix, png_files) json_local_paths <- file.path(local_cache_prefix, symbology_files) tiff_local_path <- file.path(local_cache_prefix, paste0(flow_type, "_", taxa, ".tif")) @@ -198,13 +171,13 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) ) log_progress(paste0("Cached result for week ", pred_weeks[i], ": url=", result[[i]]$url, ", legend=", result[[i]]$legend)) } - log_progress(if (save_local) "Returned cached result from localtmp" else "Returned cached result from S3") + log_progress(if (save_local) "Returned cached result from local_temp_path" else "Returned cached result from S3") return( list( start = list(week = week, taxa = taxa, loc = loc), status = "cached", result = result, - geotiff = if (save_local) tiff_local_path else paste0(s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") + geotiff = if (save_local) tiff_local_path else paste0(s3_cfg$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") ) ) } @@ -212,8 +185,8 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) # Continue with prediction if (save_local || !s3_enabled) { - dir.create("localtmp", showWarnings = FALSE) - out_path <- file.path("localtmp", gsub("/", "_", cache_prefix)) + dir.create(local_temp_path, showWarnings = FALSE) + out_path <- file.path(local_temp_path, gsub("/", "_", cache_prefix)) dir.create(out_path, recursive = TRUE, showWarnings = FALSE) } else { out_path <- tempfile(pattern = "flow_", tmpdir = "/dev/shm") @@ -265,25 +238,25 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) log_progress("Projecting and cropping raster for web output.") log_progress(paste0("combined class: ", class(combined))) - log_progress(paste0("ai_app_crs$input: ", ai_app_crs$input)) - log_progress(paste0("ai_app_extent: ", ai_app_extent)) + log_progress(paste0("s3_cfg$ai_app_crs$input: ", s3_cfg$ai_app_crs$input)) + log_progress(paste0("s3_cfg$ai_app_extent: ", s3_cfg$ai_app_extent)) if (is.null(combined) || !inherits(combined, "SpatRaster")) { log_progress("ERROR: combined raster is NULL or not a SpatRaster. Aborting.") return(format_error("combined raster is NULL or not a SpatRaster")) } - if (is.null(ai_app_crs$input)) { - log_progress("ERROR: ai_app_crs$input is NULL. Aborting.") - return(format_error("ai_app_crs$input is NULL")) + if (is.null(s3_cfg$ai_app_crs$input)) { + log_progress("ERROR: s3_cfg$ai_app_crs$input is NULL. Aborting.") + return(format_error("s3_cfg$ai_app_crs$input is NULL")) } - if (is.null(ai_app_extent)) { - log_progress("ERROR: ai_app_extent is NULL. Aborting.") - return(format_error("ai_app_extent is NULL")) + if (is.null(s3_cfg$ai_app_extent)) { + log_progress("ERROR: s3_cfg$ai_app_extent is NULL. Aborting.") + return(format_error("s3_cfg$ai_app_extent is NULL")) } log_progress("Projecting") - web_raster <- combined |> terra::project(ai_app_crs$input) + web_raster <- combined |> terra::project(s3_cfg$ai_app_crs$input) log_progress("Cropping") - web_raster <- terra::crop(web_raster, ai_app_extent) + web_raster <- terra::crop(web_raster, s3_cfg$ai_app_extent) log_progress("Done cropping") png_paths <- file.path(out_path, png_files) symbology_paths <- file.path(out_path, symbology_files) @@ -371,7 +344,7 @@ flow <- function(loc, week, taxa, n, direction = "forward", save_local = FALSE) start = list(week = week, taxa = taxa, loc = loc), status = "success", result = result, - geotiff = if (save_local) tiff_path else paste0(s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") + geotiff = if (save_local) tiff_path else paste0(s3_cfg$s3_flow_url, cache_prefix, flow_type, "_", taxa, ".tif") ) ) } diff --git a/R/globals.R b/R/globals.R deleted file mode 100644 index d911a16..0000000 --- a/R/globals.R +++ /dev/null @@ -1,18 +0,0 @@ -# Define extent of exported data (ai_app_extent) -ai_app_crs <- sf::st_crs("EPSG:3857") - -# Note now storing extent as a simple vector (xmin, xmax, ymin, ymax) in EPSG:3857 -ai_app_extent <- c(-18924313.4348565, -5565974.53966368, - 1118889.97485796, 15538711.0963092) - -# Configure S3 bucket, its url and the path within it used for flow output -s3_bucket_name <- "avianinfluenza" -s3_flow_path <- "flow/" -s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" - - -# Define local cache for temporary output images -# Will then be copied to AWS -local_cache <- tempdir() -if(!file.exists(local_cache)) - dir.create(local_cache) diff --git a/R/s3_config.R b/R/s3_config.R new file mode 100644 index 0000000..4c88da8 --- /dev/null +++ b/R/s3_config.R @@ -0,0 +1,46 @@ +s3_config <- new.env() + +set_s3_config <- function(access_key = NULL, secret_key = NULL, region = NULL, bucket = NULL, log = TRUE, log_file_path = "./flow_debug.log", local_temp_path = "localtmp") { + s3_config$access_key <- access_key + s3_config$secret_key <- secret_key + s3_config$region <- region + s3_config$bucket <- bucket + + s3_config$ai_app_crs <- sf::st_crs("EPSG:3857") + s3_config$ai_app_extent <- c(-18924313.4348565, -5565974.53966368, 1118889.97485796, 15538711.0963092) + s3_config$s3_bucket_name <- "avianinfluenza" + s3_config$s3_flow_path <- "flow/" + s3_config$s3_flow_url <- "https://avianinfluenza.s3.us-east-2.amazonaws.com/flow/" + s3_config$local_cache <- tempdir() + if(!file.exists(s3_config$local_cache)) + dir.create(s3_config$local_cache) + + s3_config$log <- log + s3_config$log_file_path <- log_file_path + s3_config$local_temp_path <- local_temp_path +} + +get_s3_config <- function() { + list( + access_key = s3_config$access_key %||% Sys.getenv("AWS_ACCESS_KEY_ID", unset = NA), + secret_key = s3_config$secret_key %||% Sys.getenv("AWS_SECRET_ACCESS_KEY", unset = NA), + region = s3_config$region %||% Sys.getenv("AWS_DEFAULT_REGION", unset = NA), + bucket = s3_config$bucket %||% Sys.getenv("S3_BUCKET_NAME", unset = NA), + ai_app_crs = s3_config$ai_app_crs %||% NA, + ai_app_extent = s3_config$ai_app_extent %||% NA, + s3_bucket_name = s3_config$s3_bucket_name %||% NA, + s3_flow_path = s3_config$s3_flow_path %||% NA, + s3_flow_url = s3_config$s3_flow_url %||% NA, + local_cache = s3_config$local_cache %||% NA, + log = s3_config$log %||% NA, + log_file_path = s3_config$log_file_path %||% NA, + local_temp_path = s3_config$local_temp_path %||% NA + ) +} + +`%||%` <- function(a, b) { + if (is.null(a)) return(b) + if (is.atomic(a) && length(a) == 1 && is.na(a)) return(b) + if (is.character(a) && length(a) == 1 && !nzchar(a)) return(b) + a +} diff --git a/man/get_s3_config.Rd b/man/get_s3_config.Rd deleted file mode 100644 index 71b2e21..0000000 --- a/man/get_s3_config.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/flow.R -\name{get_s3_config} -\alias{get_s3_config} -\title{Get S3 configuration from explicit config, environment variables, or IAM role} -\usage{ -get_s3_config() -} -\description{ -Get S3 configuration from explicit config, environment variables, or IAM role -} diff --git a/man/grapes-or-or-grapes.Rd b/man/grapes-or-or-grapes.Rd deleted file mode 100644 index 6c41ad5..0000000 --- a/man/grapes-or-or-grapes.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/flow.R -\name{\%||\%} -\alias{\%||\%} -\title{Null coalescing operator for config values} -\usage{ -a \%||\% b -} -\description{ -Null coalescing operator for config values -} diff --git a/man/set_s3_config.Rd b/man/set_s3_config.Rd deleted file mode 100644 index c97a5e1..0000000 --- a/man/set_s3_config.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/flow.R -\name{set_s3_config} -\alias{set_s3_config} -\title{Set S3 configuration explicitly} -\usage{ -set_s3_config( - access_key = NULL, - secret_key = NULL, - region = NULL, - bucket = NULL -) -} -\description{ -Allows user to set S3 credentials and bucket at runtime. If not set, environment variables or IAM role will be used. -} diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index d6b67ae..f4be360 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1 +1,2 @@ -load_models() \ No newline at end of file +load_models() +set_s3_config()