Remove model_dir requirement, major claeaning, for v1.0.0 with same function as before package

Now uses witchtools functions and mappings instead, and added historical data for standard region definitions.

Author: JohannesEmm

.gitignore

@@ -33,3 +33,9 @@ iiasa_credentials.yml
 # Development files (optional - remove these lines to track them)
 dev_workflow.R
 ROXYGEN_WORKFLOW.md
+test*
+
+# Source historical data (not released with package)
+# Only processed regional files in data/ are released
+data-raw/data_historical_values.gdx
+

DESCRIPTION

@@ -45,4 +45,4 @@ Suggests:
     tidytidbits,
     gdxrrw,
     testthat (>= 3.0.0)
-RoxygenNote: 7.2.0
+RoxygenNote: 7.3.3

R/RICE50x_plots.R

@@ -90,7 +90,7 @@ Mitigation_Sources <- function(regions=witch_regions, scenario_stringency_order,
   MITIGATION_SOURCES <- subset(MITIGATION_SOURCES, file!=scenario_stringency_order[1])
   MITIGATION_SOURCES <- MITIGATION_SOURCES[order(match(MITIGATION_SOURCES$file,scenario_stringency_order)),]
   MITIGATION_SOURCES <- MITIGATION_SOURCES[order(match(MITIGATION_SOURCES$file,scenario_stringency_order),match(MITIGATION_SOURCES$source,emi_sources)) ,]
-  MITIGATION_SOURCES$pathdir <- basename(fullpathdir[1]) #to avoid issues when saving data as EXCEL
+  MITIGATION_SOURCES$pathdir <- basename(results_dir[1]) #to avoid issues when saving data as EXCEL
   #to set minimal negative vales to zero
   MITIGATION_SOURCES$value <- pmax(MITIGATION_SOURCES$value, 0)
   #Stacked Regions Plot

R/add_historical_values.R

@@ -1,7 +1,7 @@
 
 
 Primary_Energy_Mix <- function(PES_y="value", regions="World", years=seq(yearmin, yearmax), plot_type="area", scenplot=scenlist, plot_name="Primary Energy Mix", add_total_tpes = F){
-  if(length(fullpathdir)!=1){stop("PES mix REGIONAL only for one directory at a time!"); return(invisible(NULL))}
+  if(length(results_dir)!=1){stop("PES mix REGIONAL only for one directory at a time!"); return(invisible(NULL))}
     Q_FUEL <- get_witch("Q_FUEL"); Q_FUEL_pes <- Q_FUEL %>% mutate(value=value*0.0036) %>% rename(j=fuel)
     #if fuel==uranium multiply by the efficiency of 0.3333
     Q_FUEL_pes <- Q_FUEL_pes %>% mutate(value=ifelse(j=="uranium", value*0.3333, value))
@@ -57,7 +57,7 @@ Primary_Energy_Mix <- function(PES_y="value", regions="World", years=seq(yearmin
 
 
 Electricity_Mix <- function(Electricity_y="value", regions="World", years=seq(yearmin, yearmax), plot_type="area", plot_name="Electricity Mix", scenplot=scenlist, add_total_elec=F){
-  if(length(fullpathdir)!=1){stop("Electricity mix only for one directory at a time!"); return(invisible(NULL))}
+  if(length(results_dir)!=1){stop("Electricity mix only for one directory at a time!"); return(invisible(NULL))}
     Q_IN <- get_witch("Q_IN"); Q_IN_el <- Q_IN %>% mutate(value=value * 0.0036)
     csi_el <- get_witch("csi") %>% rename(csi=value) %>% mutate(jfed=gsub("_new", "", jfed)) %>% filter(jfed %in% c("eloil", "elpb", "elpc", "elgastr", "elbigcc", "elcigcc", "elgasccs", "elpc_ccs", "elpc_oxy"))
     JFED <- merge(Q_IN_el, csi_el, by = c("t", "n", file_group_columns, "pathdir", "fuel", "jfed"), all=TRUE)
@@ -203,7 +203,7 @@ Investment_Plot <- function(regions=witch_regions, scenplot=scenlist, match_hist
 
 
 Power_capacity <- function(regions="World", years=seq(yearmin, yearmax), plot_name="Power Capacity", scenplot=scenlist){
-  if(length(fullpathdir)!=1){print("Electricity mix only for one directory at a time!")}else{
+  if(length(results_dir)!=1){print("Electricity mix only for one directory at a time!")}else{
     K_EN <- get_witch("K_EN")
     K_EN <- K_EN %>% filter(jreal %in% c("eloil", "elpb", "elpc", "elgastr", "elbigcc", "elcigcc", "elgasccs", "elpc_ccs", "elpv", "elcsp", "elnuclear", "elwindon", "elwindoff", "elhydro"))
     K_EN <- K_EN %>% mutate(category = dplyr::recode(jreal, !!!c("elpc" = "Coal w/o CCS", "elpc_ccs" = "Coal w/ CCS", "elgastr" = "Gas w/o CCS", "elgasccs" = "Gas w/ CCS", "eloil" = "Oil", "elnuclear" = "Nuclear", "elpb" = "Biomass w/o CCS", "elbigcc" = "Biomass w/ CCS", "elhydro" = "Hydro", "elwindon" = "Wind Onshore", "elwindoff" = "Wind Offshore", "elpv" = "Solar PV", "elcsp" = "Solar CSP")))

R/emission_plots.R

@@ -11,18 +11,18 @@
 #' @keywords internal
 .load_gdx_files <- function() {
   # Check if we should load GDX files
-  if(!exists("fullpathdir", envir=.GlobalEnv) ||
-     is.null(get("fullpathdir", envir=.GlobalEnv)) ||
+  if(!exists("results_dir", envir=.GlobalEnv) ||
+     is.null(get("results_dir", envir=.GlobalEnv)) ||
      exists("iamc_filename", envir=.GlobalEnv) ||
      exists("iamc_databasename", envir=.GlobalEnv)) {
     return(invisible(NULL))
   }
 
-  fullpathdir <- get("fullpathdir", envir=.GlobalEnv)
+  results_dir <- get("results_dir", envir=.GlobalEnv)
 
   # Discover GDX files
   filelist <- gsub(".gdx", "", list.files(
-    path = fullpathdir[1],
+    path = results_dir[1],
     full.names = FALSE,
     pattern = "*.gdx",
     recursive = FALSE
@@ -99,7 +99,7 @@
       exists("batch_extract", where = asNamespace("gdxtools"), mode = "function")) {
     tlen_values <- suppressWarnings(gdxtools::batch_extract(
       "tlen",
-      files = file.path(fullpathdir, paste0(filelist, ".gdx"))
+      files = file.path(results_dir, paste0(filelist, ".gdx"))
     ))
     flexible_timestep <- length(unique(tlen_values$tlen$value)) > 1
   } else {
@@ -125,15 +125,15 @@
   }
 
   # Get variable descriptions from first file
-  mygdx <- gdxtools::gdx(file.path(fullpathdir[1], paste0(filelist[1], ".gdx")))
+  mygdx <- gdxtools::gdx(file.path(results_dir[1], paste0(filelist[1], ".gdx")))
   all_var_descriptions <- rbind(
     data.frame(name = mygdx$variables$name, description = mygdx$variables$text),
     data.frame(name = mygdx$parameters$name, description = mygdx$parameters$text)
   )
   assign("all_var_descriptions", all_var_descriptions, envir=.GlobalEnv)
 
   # Set up region information
-  .setup_region_info(filelist, fullpathdir)
+  .setup_region_info(filelist, results_dir)
 
   # filelist already assigned earlier
   invisible(NULL)
@@ -142,9 +142,9 @@
 #' Setup region information
 #'
 #' @param filelist Character vector of GDX filenames
-#' @param fullpathdir Path to results directory
+#' @param results_dir Path to results directory
 #' @keywords internal
-.setup_region_info <- function(filelist, fullpathdir) {
+.setup_region_info <- function(filelist, results_dir) {
   # Get region ID
   if(!exists("reg_id", envir=.GlobalEnv)) {
     conf <- get_witch("conf")
@@ -155,7 +155,7 @@
       print("Be careful: not all results files were run with the same regional aggregation!")
     }
     scenlist <- get("scenlist", envir=.GlobalEnv)
-    reg_id <- subset(conf, file == scenlist[1] & pathdir == basename(fullpathdir[1]) & V1 == "regions")$V2
+    reg_id <- subset(conf, file == scenlist[1] & pathdir == basename(results_dir[1]) & V1 == "regions")$V2
     assign("reg_id", reg_id, envir=.GlobalEnv)
   } else {
     reg_id <- get("reg_id", envir=.GlobalEnv)
@@ -164,7 +164,7 @@
   # Get regions list
   if (requireNamespace("gdxtools", quietly = TRUE) &&
       exists("batch_extract", where = asNamespace("gdxtools"), mode = "function")) {
-    n <- suppressWarnings(gdxtools::batch_extract("n", files = file.path(fullpathdir, paste0(filelist, ".gdx"))))
+    n <- suppressWarnings(gdxtools::batch_extract("n", files = file.path(results_dir, paste0(filelist, ".gdx"))))
     if(is.null(n$n)) {
       witch_regions <- "World"
     } else {
@@ -173,7 +173,7 @@
   } else {
     # Fallback: try to get regions from first file
     tryCatch({
-      first_gdx <- gdxtools::gdx(file.path(fullpathdir[1], paste0(filelist[1], ".gdx")))
+      first_gdx <- gdxtools::gdx(file.path(results_dir[1], paste0(filelist[1], ".gdx")))
       if("n" %in% names(first_gdx$sets)) {
         witch_regions <- first_gdx$sets$n$V1
       } else {
@@ -194,28 +194,6 @@
   assign("display_regions", display_regions, envir=.GlobalEnv)
   assign("witch_regions", witch_regions, envir=.GlobalEnv)
 
-  # Check for historical data directory - try to find actual directory
-  model_dir <- get("model_dir", envir=.GlobalEnv)
-  if(!is.null(model_dir)) {
-    data_dir <- file.path(model_dir, paste0("data_", reg_id))
-    if(!dir.exists(data_dir)) {
-      # Try to find a directory that starts with data_<reg_id>
-      potential_dirs <- list.dirs(model_dir, full.names = FALSE, recursive = FALSE)
-      matching_dirs <- grep(paste0("^data_", reg_id), potential_dirs, value = TRUE)
-      if (length(matching_dirs) > 0) {
-        message("Found data directory: ", matching_dirs[1])
-        # Update reg_id to match actual directory
-        reg_id <- gsub("^data_", "", matching_dirs[1])
-        assign("reg_id", reg_id, envir=.GlobalEnv)
-      } else {
-        warning(sprintf(
-          "Historical data directory not found: '%s'\nHistorical data features will be disabled.",
-          data_dir
-        ))
-      }
-    }
-  }
-
   # Set up color palettes
   region_palette <- get_region_palette(witch_regions, reg_id)
   if(exists("restrict_regions", envir=.GlobalEnv)) {

R/energy_plots.R

@@ -23,8 +23,15 @@
     stop("No GDX files found in: ", results_dir)
   }
 
-  # Apply inclusion filter
-  if (!is.null(restrict_files) && restrict_files != "") {
+  # ALWAYS filter to files starting with "results_" first
+  all_files <- all_files[stringr::str_starts(all_files, "results_")]
+
+  if (length(all_files) == 0) {
+    stop("No GDX files starting with 'results_' found in: ", results_dir)
+  }
+
+  # Apply additional inclusion filters (if restrict_files is not "results_")
+  if (!is.null(restrict_files) && restrict_files != "" && restrict_files != "results_") {
     patterns <- if (is.character(restrict_files)) restrict_files else unlist(restrict_files)
     filtered <- all_files
     for (pattern in patterns) {
@@ -82,7 +89,6 @@
 #' Returns a list with all session data instead of using global variables.
 #'
 #' @param results_dir Path to results directory
-#' @param model_dir Path to model directory
 #' @param restrict_files Pattern to filter GDX files
 #' @param exclude_files Pattern to exclude GDX files
 #' @param removepattern Pattern to remove from scenario names
@@ -91,7 +97,6 @@
 #' @return List containing: filelist, scenlist, regions, palettes, metadata
 #' @keywords internal
 .load_gdx_session <- function(results_dir,
-                              model_dir = NULL,
                               restrict_files = "results_",
                               exclude_files = "",
                               removepattern = "results_",
@@ -129,23 +134,22 @@
   ))
 
   # Set filelist globally early so get_witch() can access it
-  # This is needed because get_witch() depends on filelist being global
+  # This is needed because get_witch() depends on this being global
+  # Note: results_dir is already assigned by the caller (run_witch, etc.)
   assign("filelist", filelist, envir = .GlobalEnv)
-  assign("fullpathdir", results_dir, envir = .GlobalEnv)
 
   # Get metadata from first file
   first_gdx_path <- file.path(results_dir, paste0(filelist[1], ".gdx"))
   metadata <- .extract_gdx_metadata(first_gdx_path, filelist, results_dir)
 
   # Get region information (suppress join messages)
-  region_info <- suppressMessages(.extract_region_info(filelist, results_dir, reg_id, model_dir))
+  region_info <- suppressMessages(.extract_region_info(filelist, results_dir, reg_id))
 
   # Return everything as a list
   list(
     filelist = filelist,
     scenlist = scenlist,
     results_dir = results_dir,
-    model_dir = model_dir,
     file_group_columns = file_group_columns,
     regions = region_info$regions,
     reg_id = region_info$reg_id,
@@ -212,10 +216,9 @@
 #' @param filelist GDX filenames
 #' @param results_dir Results directory
 #' @param reg_id Regional aggregation ID
-#' @param model_dir Model directory
 #' @return List with region info and palettes
 #' @keywords internal
-.extract_region_info <- function(filelist, results_dir, reg_id = NULL, model_dir = NULL) {
+.extract_region_info <- function(filelist, results_dir, reg_id = NULL) {
   # Determine reg_id if not provided
   if (is.null(reg_id)) {
     # Try to read conf directly from the first GDX file
@@ -248,24 +251,6 @@
   # Get regions from GDX files
   regions <- .get_regions_from_gdx(filelist, results_dir)
 
-  # Check for historical data directory - try to find actual directory
-  if (!is.null(model_dir) && length(model_dir) > 0 && !is.null(reg_id) && length(reg_id) > 0) {
-    data_dir <- file.path(model_dir, paste0("data_", reg_id[1]))
-    if (!dir.exists(data_dir)) {
-      # Try to find a directory that starts with data_<reg_id>
-      potential_dirs <- list.dirs(model_dir, full.names = FALSE, recursive = FALSE)
-      matching_dirs <- grep(paste0("^data_", reg_id[1]), potential_dirs, value = TRUE)
-      if (length(matching_dirs) > 0) {
-        data_dir <- file.path(model_dir, matching_dirs[1])
-        message("Found data directory: ", matching_dirs[1])
-        # Update reg_id to match actual directory
-        reg_id <- gsub("^data_", "", matching_dirs[1])
-      } else {
-        warning("Historical data directory not found: ", data_dir,
-                "\nHistorical data features will be disabled.")
-      }
-    }
-  }
 
   # Create color palettes
   palette <- get_region_palette(regions, reg_id)