Makefile

# Makefile for research workflow and installing necessary libraries
#
# Copyright (c) 2020-2025 Karel Kaurila
#
# -----------------------------------------------
# This project has been run with the following versions for GDAL, PROJ and GEOS
# Please ensure these are installed and found within PATH
GDAL_VERSION := 3.4.1
PROJ_VERSION := 8.2.1
GEOS_VERSION := 3.10.2
# Paths to the libraries above:
GDAL_CONFIG := $(shell which gdal-config)
PROJ := $(shell which proj)
GEOS_CONFIG := $(shell which geos-config)
# -------------------------------------------------------

# Contents
# --------------------------
# Set common environmental variables and paths ....... general_paths
# Canned recipes and functions ....................... recipes_functions

# Install libraries and download data
# Install Python ..................................... install_python
# Compile R within project folder .................... compile_R
# Install R packages ................................. r_packages

# Model fitting and results .......................... model_fitting
# Figures and summaries for model fit ................ figures_summaries
# Posterior sampling ................................. posterior_sampling
# Model comparison metrics ........................... comparison_metrics
# Sensitivity analysis  .............................. sensitivity_to_samplesize
# Simplified sensitivity analysis  ................... sensitivity_simple
# Prior sensitivity analysis  ........................ sensitivity_prior

# Figures
# Manuscript figures  ................................ manuscript_figures
# Miscellaneous figures in supplement ................ supplement_figures

# Create folders ..................................... mk_dirs

# ---------------------------------------------------------------
# Set common environmental variables and paths | general_paths

PROJECT_ROOT := $(shell pwd)
this_file = $(lastword $(MAKEFILE_LIST))

# optional preamble makefile
-include preambs.mk
# optional file containing list of folders with recent results
RESULT_DIRS_FILE := model_dirs.ini
ifneq ("$(wildcard $(RESULT_DIRS_FILE))","")
  RESULT_DIRS := $(file < $(RESULT_DIRS_FILE))
else
  RESULT_DIRS := results/example_results
endif

# collect all folder paths
DIRS += $(RESULT_DIRS)

# compile R within the project folder?
COMPILE_R = y

BIN := bin
DATA_DIR := data
VENV := $(BIN)/.venv
VENV_BIN := $(VENV)/bin
PY_INSTALL_STAMP := $(VENV)/.install.stamp
PYTHON := $(VENV_BIN)/python
PY3 := $(shell command -v python3 2> /dev/null)
DIRS += $(DATA_DIR)

R_VERS := 4.2.2
R_TARBALL := R-$(R_VERS).tar.gz
R_ARCHIVE := $(BIN)/$(R_TARBALL)
R_URL := https://cloud.r-project.org/src/base/R-4/$(R_TARBALL)
LOCAL_RDIR := $(BIN)/R-$(R_VERS)
R_PREFIX := $(PROJECT_ROOT)/$(LOCAL_RDIR)
R_BASE_STAMP := $(R_PREFIX)/.install.stamp
R_CONFIG_OPT := --enable-R-shlib --prefix $(R_PREFIX)
R_FLAGS := --no-save
# .R source files for this project
SRC_DIR = src
# .R files that are executed from this Makefile as scripts
SCRIPT_DIR := $(SRC_DIR)/scripts

.DEFAULT_GOAL := all

.PHONY : all

all : install result_files sensitivity supplement

.PHONY: help
help:
	@echo "Usage: make <target>, where <target> is one of the following:"
	@echo ""
	@echo "help         print this message"
	@echo "install      install all libraries"
	@echo "fit_model    fit Integrated Species Distribution Model to data"
	@echo "result_files model fit summary tables and figures"
	@echo "sensitivity  run sensitivity analysis with increasing amounts of training data"
	@echo "supplement   Generate supplement figures in figures/ folder."
	@echo "all          fit_model, sensitivity and supplement"
	@echo "variables    (debugging target) print contents for some Makefile variables"
	@echo ""
	@echo "You can set the folder used as the basis for results with the argument"
	@echo "RESULTS_DIR=""<directory>"" including the quotes."


install: $(PY_INSTALL_STAMP)
ifeq ($(COMPILE_R),y)
  install: $(R_BINARIES)
endif

.PHONY: build
build:

.PHONY: clean


DIRS += $(BIN)

#-------------------------------------------
# Canned recipes and functions | recipes_functions
#-------------------------------------------

# Update time stamp of an existing file
# (intentionally) throws error if file does not exist
# used for recipes that produce multiple files from one invocation as a
# substitute for the &: operator that does not exist in older versions of Make
update_stamp = test -f $1 && touch $1

# extract parent folder name
parent_name = $(lastword $(subst /, ,$(dir $(abspath $1))))
just_dir = $(patsubst %/,%,$(dir $1))

# extract path to parent folder
parent_dir = $(dir ${1:/=})
gpar_dir = $(call parent_dir, $(call parent_dir, $1))


# temporary file for target
temp_target = $(addsuffix $(suffix $@),$(basename $@)_tmp)

# generate paths dir1/file1 dir1/file2 ... dir1/file[m] dir2/file1 ... dir[n]/file[m]
# where first argument is list of dirs (dir1 dir2 ... dir[n])
# and second argument is list of files (file1 file2 ... file[m])
dirs_files = $(foreach dir, $1, $(addprefix $(dir)/,$2))

# all words in a list except the first
# useful when multiple files created from one invocation
other_words = $(wordlist 2, $(words $1), $1)

#-------------
# Python  | install_python
#-------------
# See guide in https://blog.mathieu-leplatre.info/tips-for-your-makefile-with-python.html

$(PYTHON): | $(BIN)
	@if [ -z $(PY3) ]; then echo "python3 could not be found. See https://docs.python.org/3/"; exit 2; fi
	python3 -m venv $(VENV)

$(PY_INSTALL_STAMP): $(PYTHON) py_requirements.txt
	$(PYTHON) -m pip install -r py_requirements.txt
	touch $(PY_INSTALL_STAMP)

py_clean:
	find . -type d -name "__pycache__" | xargs rm -rf {};
	rm -rf $(VENV)
.PHONY: py_clean

clean: py_clean

# Python packages / executables

# package for downloading files from Zenodo repositories
ZENODO_GET := $(VENV_BIN)/zenodo_get

# some useful tools for .csv files from csvkit
CSVSTACK := $(VENV_BIN)/csvstack
CSVLOOK := $(VENV_BIN)/csvlook
CSVCUT := $(VENV_BIN)/csvcut
CSVFORMAT := $(VENV_BIN)/csvformat
CSVGREP := $(VENV_BIN)/csvgrep
CSVJOIN := $(VENV_BIN)/csvjoin
CSVSORT := $(VENV_BIN)/csvsort
CSVSTAT := $(VENV_BIN)/csvstat
CSVKIT = $(CSVSTACK) $(CSVLOOK) $(CSVCUT) $(CSVFORMAT) $(CSVGREP) $(CSVJOIN) \
         $(CSVSORT) $(CSVSTAT)

PY_PKGS := $(ZENODO_GET) $(CSVKIT)

# All python packages are installed based on the py_requirements.txt file
# Only need to update the time stamp for the package executable
$(PY_PKGS) : $(PY_INSTALL_STAMP)
	$(call update_stamp, $@)

# Define data

TRANSECT_FILE := transect_data.csv
TRANSECT_PATH := $(DATA_DIR)/$(TRANSECT_FILE)

ASSESSMENT_FILE := expert_assessments.tif

COVARIATE_FILE := covariate_raster_example.tif

DATA_FILE_NAMES = $(TRANSECT_FILE) $(ASSESSMENT_FILE) $(COVARIATE_FILE)
DATA_FILES = $(addprefix $(DATA_DIR)/, $(DATA_FILE_NAMES))

# Download data automatically from Zenodo
DATA_RECORD_FILE := $(DATA_DIR)/zenodo_records.txt
DATA_MD5SUM := $(DATA_DIR)/md5sums.txt

.PHONY: datafiles
datafiles: $(DATA_FILES)

$(DATA_FILES): $(DATA_MD5SUM)
	$(call update_stamp, $@)

$(DATA_MD5SUM): $(DATA_RECORD_FILE) | $(DATA_DIR) $(PY_INSTALL_STAMP)
	$(ZENODO_GET) $(file < $<) -m -o $(DATA_DIR)
	cd $(DATA_DIR) && md5sum -c md5sums.txt


#----------------
# compile_R
#----------------
# Install R locally to the project folder by compiling from source
# see: https://www.reddit.com/r/Rlanguage/comments/199i52k/how_to_install_an_older_version_of_r_412_or_older/

R_BIN := $(LOCAL_RDIR)/bin

R_SCRIPT_LOCAL := $(R_BIN)/Rscript
R_SCRIPT_DEF := $(shell which Rscript)
R_EXEC_LOCAL := $(R_BIN)/R
R_EXEC_DEF := $(shell which R)
R_BINARIES = $(R_SCRIPT_LOCAL) $(R_EXEC_LOCAL)

ifeq ($(COMPILE_R),y)
  R_SCRIPT := $(R_SCRIPT_LOCAL)
  R_EXEC := $(R_EXEC_LOCAL)
else
  R_SCRIPT := $(R_SCRIPT_DEF)
  R_EXEC := $(R_EXEC_DEF)
endif
R_EXPR := $(R_EXEC) $(R_FLAGS) -e

RENV_LOCK := renv.lock
RENV_STAMP := $(LOCAL_RDIR)/.renv.stamp

TAR_EXTRACT := tar zxfv


$(LOCAL_RDIR): | $(R_ARCHIVE)
	cd $(BIN) && $(TAR_EXTRACT) $(R_TARBALL);

$(R_ARCHIVE): | $(BIN)
	cd $(BIN) && wget -N $(R_URL)

# All the binaries are produced by applying this recipe once
# The message during a dry-run assumes R binaries would be compiled
$(R_BINARIES) : | $(LOCAL_RDIR)
ifneq (,$(findstring n,$(firstword -$(MAKEFLAGS))))
	$(info "dry-run: would perform following recursive make:")
	$(info "cd $(LOCAL_RDIR) && ./configure $(R_CONFIG_OPT)")
	$(info "$(MAKE) -C $(R_PREFIX)")
	$(info "$(MAKE) -C $(R_PREFIX) install")
else ifeq ($(COMPILE_R),y)
	cd $(LOCAL_RDIR) && ./configure $(R_CONFIG_OPT)
	$(MAKE) -C $(R_PREFIX)
	$(MAKE) -C $(R_PREFIX) install
endif


# ---------------------------------------------------------------
# r_packages
# Install R packages using R package 'renv'

RENV_SCRIPT := $(SCRIPT_DIR)/restore_packages.R

$(RENV_STAMP): $(RENV_LOCK) | $(R_SCRIPT)
	$(R_SCRIPT) $(RENV_SCRIPT)
	touch $(RENV_STAMP)

install: $(RENV_STAMP)

# Shortcut to local R binary (and Rscript)
# Useful for testing things in the local R environment
# Use with ./R
R_SHORTCUT = R
RSCRIPT_SHORTCUT = Rscript

$(R_SHORTCUT) : $(R_EXEC)
	ln -s $(R_EXEC) $(R_SHORTCUT)

$(RSCRIPT_SHORTCUT) : $(R_SCRIPT)
	ln -s $(R_SCRIPT) $(RSCRIPT_SHORTCUT)


ifeq ($(COMPILE_R),y)
install : $(R_SHORTCUT) $(RSCRIPT_SHORTCUT)
endif

.PHONY: r_clean
r_clean:
	if [ -d $(LOCAL_RDIR) ]; then rm -r $(LOCAL_RDIR); fi
	if [ -L $(R_SHORTCUT) ]; then rm $(R_SHORTCUT); fi
	if [ -L $(RSCRIPT_SHORTCUT) ]; then rm $(RSCRIPT_SHORTCUT); fi
clean: r_clean

#--------------------------------------------------
# Model fitting and results | model_fitting
#--------------------------------------------------
# The general model fitting recipe is built around three files:
# 1. parameters.RData  - likelihood families, prior hyperparameters, which data sets to use
# 2. data.RData        - data for fitting the model
# 3. model.RData       - the resulting model, including posterior distributions
#                        for parameters
# Each model fit has its own results folder containing these files.
# The folder may also contain other files derived from the three files above
# for summarising the results, such as figures or tables.

RESULTS_ROOT := results
# Default to results from example model
# Change variable below to generate result files etc. for other results
RESULTS_DIR := $(RESULTS_ROOT)/example_results

DIRS += $(RESULTS_DIR)

#
PARAM_FILE_NAME := parameters.RData

# Covariate raster file used in model fitting, defaults to the file downloaded
# from Zenodo
COV_RAST_FILE ?= $(DATA_DIR)/$(COVARIATE_FILE)
PARAM_FILE_ARGS = $(PARAM_SCRIPT) -c $(COV_RAST_FILE)

# Script for setting and saving model fitting parameters
PARAM_SCRIPT := $(SCRIPT_DIR)/parameters_script.R
PARAM_MAIN := $(SRC_DIR)/ISDM_parameters.R

%/$(PARAM_FILE_NAME) : $(PARAM_SCRIPT) $(PARAM_MAIN) | % $(R_SCRIPT)
	$(R_SCRIPT) $(PARAM_FILE_ARGS) -o $(@D)

# Define each model in model comparison with template files
PARAM_TEMPLATE_SCRIPT := $(SCRIPT_DIR)/model_param_template.R
PARAM_TEMPLATE_DIR := templates
DIRS += $(PARAM_TEMPLATE_DIR)

TEMPLATE_MODELS := nb_beta nb_bernoulli bernoulli_beta bernoulli_bernoulli
TEMPLATE_FILES = $(addsuffix .RData, $(addprefix $(PARAM_TEMPLATE_DIR)/, $(TEMPLATE_MODELS)))

install: $(TEMPLATE_FILES)

$(TEMPLATE_FILES): $(PARAM_TEMPLATE_DIR)/%.RData : $(PARAM_TEMPLATE_SCRIPT) | $(R_SCRIPT) $(PARAM_TEMPLATE_DIR)
	$(R_SCRIPT) $(PARAM_TEMPLATE_SCRIPT) $@ $* $(COV_RAST_FILE)

MODEL_COMP_ROOT := $(RESULTS_ROOT)/model_comparison
$(MODEL_COMP_ROOT)/%/parameters.RData : $(PARAM_SCRIPT)\
  | $(MODEL_COMP_ROOT)/% $(R_SCRIPT)
	$(R_SCRIPT) $(PARAM_FILE_ARGS) -o $(@D) -m $*

# Recipe for fitting each model ------------------------------
MODEL_SCRIPT := $(SCRIPT_DIR)/model_script.R
MODEL_MAIN := $(SRC_DIR)/fit_ISDM.R

MODEL_FILES := model.RData data.RData parameters.RData
MODEL_FILE_PATHS := $(call dirs_files, $(RESULT_DIRS),$(MODEL_FILES))

$(RESULTS_ROOT)/%/model.RData : $(DATA_FILES) $(RESULTS_ROOT)/%/parameters.RData\
  $(RESULTS_ROOT)/%/data.RData $(MODEL_SCRIPT) $(MODEL_MAIN) | $(R_SCRIPT)
	$(R_SCRIPT) $(MODEL_SCRIPT) $(@D)/parameters.RData $(@D)/data.RData

CNSTR_DATA_MAIN := $(SRC_DIR)/ISDM_data.R
CNSTR_DATA_SCRIPT := $(SCRIPT_DIR)/data_script.R
CNSTR_DATA_DEP = $(CNSTR_DATA_MAIN) $(CNSTR_DATA_SCRIPT)

# Separate target for preprocessing data.RData
.PHONY: preprocess_data
preprocess_data: $(addsuffix /data.RData, $(RESULT_DIRS))


# Separate target for parameter files
.PHONY: param_files
param_files : $(addsuffix /parameters.RData, $(RESULT_DIRS))

.PHONY : fit_model
fit_model : $(MODEL_FILE_PATHS)


# ---------------------------------------------------
# Figures and summaries for model fit | figures_summaries

FIGURE_SCRIPT = $(SCRIPT_DIR)/plot_result_figure.R
POINTRANGE_SCRIPT = $(SCRIPT_DIR)/pointrange_plots.R

# Original figure names before renaming
SURVEY_FIGS := 1_survey_pred_mean.png 1_survey_pred_sd.png

BYM_FIG = bym_mu.png
BYM_FIG_PATHS = $(call dirs_files, $(RESULT_DIRS), $(BYM_FIG))

VIOLIN_FIGS = 2_violin.png 3_exp_coeff.png

# Pointrange figures
PARAM_PORA = parameter_pointrange.png
SKILL_PORA = exp_skill_pointrange.png
PORA_FIGS = $(PARAM_PORA) $(SKILL_PORA)
PORA_FIG_PATHS = $(call dirs_files, $(RESULT_DIRS), $(PORA_FIGS))

MAP_FIGS := $(SURVEY_FIGS) $(BYM_FIG)
RESULT_FIG_PATHS := $(addprefix $(RESULTS_DIR)/, $(MODEL_FIGS))


.PHONY : result_figures

FIGURE_FILES := $(SURVEY_FIGS) $(PORA_FIGS) $(BYM_FIG)
FIG_PATHS := $(call dirs_files, $(RESULT_DIRS), $(FIGURE_FILES))

result_figures : $(FIG_PATHS)

# Arrange experts by median skill coeffients for better visualization
# Save ranking in a file
SKILL_RANK_FILENAME := skill_ranking.txt
# Use nb_beta model as reference
REFERENCE_MODEL_DIR := results/model_comparison/nb_beta
SKILL_RANK_REFERENCE := $(REFERENCE_MODEL_DIR)/$(SKILL_RANK_FILENAME)


.PHONY: skill_ranking
skill_ranking: $(SKILL_RANK_REFERENCE)

$(SKILL_RANK_REFERENCE): $(addprefix $(REFERENCE_MODEL_DIR)/,$(MODEL_FILES)) | $(POINTRANGE_SCRIPT)
	$(R_SCRIPT) $(POINTRANGE_SCRIPT) $(@D) 'ranking' $@

%/$(SKILL_RANK_FILENAME): $(SKILL_RANK_REFERENCE)
	cp -u $(SKILL_RANK_REFERENCE) $@


# Common dependencies between figures
$(FIG_PATHS) : | $(R_SCRIPT)

$(RESULT_FIG_PATHS) : $(FIGURE_SCRIPT)

$(PORA_FIG_PATHS) : $(POINTRANGE_SCRIPT)


%/1_survey_pred_mean.png : %/model.RData %/data.RData %/parameters.RData | $(R_SCRIPT)
	$(R_SCRIPT) $(FIGURE_SCRIPT) $(@D) 'survey_pred'

# survey_pred_sd is built with the same invocation as survey_pred_mean
%/1_survey_pred_sd.png : %/1_survey_pred_mean.png
	$(call update_stamp, $@)

%/2_violin.png : %/model.RData %/data.RData %/parameters.RData
	$(R_SCRIPT) $(FIGURE_SCRIPT) $(@D) 'violin'

%/3_exp_coeff.png : %/model.RData %/data.RData %/parameters.RData
	$(R_SCRIPT) $(FIGURE_SCRIPT) $(@D) 'exp_coeff'

%/$(PARAM_PORA): %/model.RData %/data.RData %/parameters.RData
	$(R_SCRIPT) $(POINTRANGE_SCRIPT) $(@D) 'params'

%/$(SKILL_PORA): %/model.RData %/data.RData %/parameters.RData %/$(SKILL_RANK_FILENAME)
	$(R_SCRIPT) $(POINTRANGE_SCRIPT) $(@D) 'skill' $(@D)/$(SKILL_RANK_FILENAME)

%/$(BYM_FIG) : %/model.RData %/data.RData %/parameters.RData $(FIGURE_SCRIPT) %/$(SKILL_RANK_FILENAME) | $(R_SCRIPT)
	$(R_SCRIPT) $(FIGURE_SCRIPT) $(@D) 'bym' $(@D)/$(SKILL_RANK_FILENAME)

# -------------------------------------------
# Posterior sampling   |  posterior_sampling

POST_SAMP_SCRIPT := $(SCRIPT_DIR)/post_pred_sampling.R
POST_SAMP_FNAME := posterior_samples.RData
POST_SAMP_FILES = $(addsuffix /$(POST_SAMP_FNAME), $(RESULT_DIRS))

.PHONY: post_samples
post_samples : $(POST_SAMP_FILES)

%/$(POST_SAMP_FNAME) : $(addprefix %/,$(MODEL_FILES)) $(POST_SAMP_SCRIPT) | $(R_SCRIPT)
	$(R_SCRIPT) $(POST_SAMP_SCRIPT) $(@D) $(basename $(POST_SAMP_FNAME))

# --------------------------------------------
# Model comparison metrics | comparison_metrics

COMPARISON_SCRIPT = $(SCRIPT_DIR)/model_comparison_script.R
COMPARISON_R_DEPS = $(SRC_DIR)/model_comparison_sampling.R
COMPARISON_NAME = comparison.csv
COMPARISON_FILE := $(addsuffix /$(COMPARISON_NAME), $(RESULT_DIRS))

# Comparison summary
COMPARISON_SUM_DIR := $(dir $(firstword $(RESULT_DIRS)))
COMPARISON_RAW_TB := $(addprefix $(COMPARISON_SUM_DIR), comparison_raw.csv)
COMPARISON_SUM_TB := $(addprefix $(COMPARISON_SUM_DIR), comparison_summary.csv)
COMPARISON_SUM_TEX := $(addprefix $(COMPARISON_SUM_DIR), comparison_summary.tex)

# Convert .csv to .tex tabular
CSV_TO_TEX_SCRIPT := $(SCRIPT_DIR)/csvToLatex.py
CSV_TO_TEX := $(PYTHON) $(CSV_TO_TEX_SCRIPT)

.PHONY : result_table

result_table : $(COMPARISON_FILE) $(COMPARISON_SUM_TB) $(COMPARISON_SUM_TEX)

%/comparison.csv : $(COMPARISON_SCRIPT) $(COMPARISON_R_DEPS) $(addprefix %/,$(MODEL_FILES) $(POST_SAMP_FNAME)) | $(R_SCRIPT)
	$(R_SCRIPT) $(COMPARISON_SCRIPT) $(@D) $(@D)/$(POST_SAMP_FNAME)

$(COMPARISON_RAW_TB) : $(COMPARISON_FILE) | $(CSVSTACK)
	$(CSVSTACK) $(COMPARISON_FILE) > $(COMPARISON_RAW_TB)

$(COMPARISON_SUM_TB) : $(COMPARISON_RAW_TB) | $(CSVCUT)
	$(CSVCUT) -c "model,cpo" $(COMPARISON_RAW_TB) > $(COMPARISON_SUM_TB)

$(COMPARISON_SUM_TEX) : $(COMPARISON_SUM_TB) $(CSV_TO_TEX_SCRIPT) | $(PY_INSTALL_STAMP)
	$(CSV_TO_TEX) "$(COMPARISON_SUM_TB)" "$(COMPARISON_SUM_TEX)"

.PHONY : result_files

result_files : result_figures result_table

#------------------------------
# sensitivity analysis  | sensitivity_to_samplesize
# Compare each model's sensitivity to amount of survey training data.
# Running all of these may require several days, depending on the computer.

SENS_DIR_NAME = sensitivity_analysis
SENS_DIR = $(addprefix $(RESULTS_ROOT)/,$(SENS_DIR_NAME))
SENS_SUB_DIRS = $(addprefix $(SENS_DIR)/,1 2 3 4)
DIRS += $(SENS_DIR) $(SENS_SUB_DIRS)

SENS_PARAMS = $(addsuffix /parameters.RData,$(SENS_SUB_DIRS))
SENS_MODEL = $(addsuffix /model.RData, $(SENS_SUB_DIRS))
SENS_DATA = $(addsuffix /data.RData, $(SENS_SUB_DIRS))
SENS_FIT = $(SENS_PARAMS) $(SENS_MODEL) $(SENS_DATA)

# Sample indices for crossvalidation folds
N_FOLDS = 5
N_PERMS = 30
N_TEST = 31
PERM_SEED = 1
# Is the last fold identical for all permutations?
# (default: no, each permutation is different)
LAST_FOLD_SAME = n

CV_FOLDS = $(shell seq $(N_FOLDS))
ifeq ($(LAST_FOLD_SAME),y)
  FOLD_STRS_ARG := $(filter-out $(lastword $(CV_FOLDS)), $(CV_FOLDS))
else
  FOLD_STRS_ARG := $(CV_FOLDS)
endif

CV_FOLD_OPT = -f $(TRANSECT_PATH) -p $(N_PERMS) -n $(N_FOLDS)\
	      --seed $(PERM_SEED) --n_test $(N_TEST)
PERM_STRS := $(addprefix perm_, $(shell seq $(N_PERMS)))
FOLD_STRS := $(addprefix fold_, $(FOLD_STRS_ARG))
CV_PERM_DIRS := $(addprefix $(SENS_DIR)/, $(PERM_STRS))
CV_FOLD_DIRS := $(call dirs_files, $(CV_PERM_DIRS), $(FOLD_STRS))

ifeq ($(LAST_FOLD_SAME),y)
  CV_FOLD_DIRS += $(SENS_DIR)/perm_1/fold_$(N_FOLDS)
endif

DIRS += $(CV_PERM_DIRS) $(CV_FOLD_DIRS)

CV_FOLD_SCRIPT = $(SCRIPT_DIR)/cvFolds.R
CV_FOLDS_TB := $(SENS_DIR)/cv_indices_$(N_PERMS).csv

$(CV_FOLDS_TB) : $(CV_FOLD_SCRIPT) | $(dir $(CV_FOLDS_TB))
	$(R_SCRIPT) $(CV_FOLD_SCRIPT) $(CV_FOLD_OPT) -o $@

# Sub tables containing for each permutation
CV_SUB_TB = $(addsuffix /folds.csv, $(CV_PERM_DIRS))

perm_col = $(patsubst perm_%,p_%,$(call parent_name, $1))


%/folds.csv : $(CV_FOLDS_TB) | % $(CSVCUT)
	$(CSVCUT) -c "id,$(call perm_col,$@)" $(CV_FOLDS_TB) > $@

.PHONY: cv_fold_tb

cv_fold_tb: $(CV_FOLDS_TB) $(CV_SUB_TB)

# Index files
# These are tables denoting which survey observation belongs in which training/testing fold
IND_SCRIPT := $(SCRIPT_DIR)/foldCsvToInd.R

fold_num = $(notdir $(subst fold_,,$(basename $1)))
perm_file = $(addprefix $(dir $(patsubst %/,%,$1 )),folds.csv)

# ind_train recipe is defined after SECONDEXPANSION
# ind_test is create by the same invocation
%/ind_test.txt : %/ind_train.txt
	$(call update_stamp, $@)

CV_IND = $(addsuffix /ind_train.txt, $(CV_FOLD_DIRS))
CV_IND += $(addsuffix /ind_test.txt, $(CV_FOLD_DIRS))

.PHONY: cv_fold_ind

cv_fold_ind : $(CV_IND)

CV_MODEL_FIT = $(call dirs_files, $(CV_FOLD_DIRS), $(MODEL_FILES))

cv_fit: $(CV_MODEL_FIT)

.PHONY: cv_samples
CV_SAMPLES = $(addsuffix /$(POST_SAMP_FNAME), $(CV_FOLD_DIRS))
cv_samples : $(CV_SAMPLES)

# Comparison tables --------------------------------------------------------

# Comparisons within each permutation * fold
CV_COMP = $(addsuffix /comparison.csv, $(CV_FOLD_DIRS))
.PHONY: cv_comparison
cv_comparison: $(CV_COMP)

# Stack comparisons within each permutation together
CV_PERM_COMP = $(addsuffix /fold_comparisons.csv, $(CV_PERM_DIRS))

.PHONY: cv_perm_csv
cv_perm_csv : $(CV_PERM_COMP)
%/fold_comparisons.csv : $(addprefix %/, $(addsuffix /comparison.csv, $(FOLD_STRS))) | $(CSVSTACK)
	$(CSVSTACK) $^ > $@


CV_PERMS := $(shell seq $(N_PERMS))

CV_COMP_RAW_TB = $(SENS_DIR)/cv_raw_$(lastword $(CV_PERMS)).csv

# Stack results within each permutation first
# Use argument -o $(CV_COMP_RAW_TB) to avoid rerunning model fitting
$(CV_COMP_RAW_TB) : $(CV_PERM_COMP) | $(CSVSTACK)
	$(CSVSTACK) $^ > $@

.PHONY: cv_raw_tb
cv_raw_tb: $(CV_COMP_RAW_TB)

CV_SUM_SCRIPT = $(SCRIPT_DIR)/cvSummary.R
CV_COMP_SUM_TB = $(SENS_DIR)/cv_sum_$(lastword $(CV_PERMS)).csv
$(CV_COMP_SUM_TB) : $(CV_COMP_RAW_TB) | $(CSVCUT) $(R_SCRIPT)
	$(R_SCRIPT) $(CV_SUM_SCRIPT) -f $< -o $(temp_target)
	$(CSVCUT) -c "model,lpd,n.train" $(temp_target) > $@


.PHONY: cv_sum_tb
cv_sum_tb: $(CV_COMP_SUM_TB)

# -----------------------------------------------------
# Simple sensitivity analysis  | sensitivity_simple
# Similar to above, but without cross validation.
# Instead, observations with replicates are split into training folds, while
# observations without replicates are used as the test set.

$(SENS_PARAMS) : $(SENS_DIR)/%/parameters.RData : $(PARAM_SCRIPT) |  $(R_SCRIPT)
	$(R_SCRIPT) $(PARAM_FILE_ARGS) -o $(@D) -t $*

SENS_TABLES = $(addsuffix /comparison.csv, $(SENS_SUB_DIRS))
SENS_RES_SCRIPT = $(SCRIPT_DIR)/collect_sensitivity_results.R
SENS_RESULT = $(addprefix $(SENS_DIR)/, sensitivity_comparison.csv)
SENS_RESULT_RAW := $(SENS_DIR)/sensitivity_comparison_raw.csv

$(SENS_RESULT_RAW) : $(SENS_TABLES) | $(CSVSTACK)
	$(CSVSTACK) $(SENS_TABLES) > $(SENS_RESULT_RAW)

$(SENS_RESULT) : $(SENS_RESULT_RAW) | $(CSVCUT)
	$(CSVCUT) -c "model,lpd,n.train" $(SENS_RESULT_RAW) > $(SENS_RESULT)

.PHONY: sensitivity

sensitivity: $(SENS_FIT) $(SENS_TABLES) $(SENS_RESULT)

# Model fitting only
.PHONY: sensitivity_fit
sensitivity_fit: $(SENS_FIT)

.PHONY: sensitivity_results
sensitivity_results: $(SENS_RESULT)

# Posterior samples
SENS_SAMPLES = $(addsuffix /$(POST_SAMP_FNAME),$(SENS_SUB_DIRS))
.PHONY: sensitivity_samples
sensitivity_samples: $(SENS_SAMPLES)

# Parameter files only
.PHONY: sensitivity_params
sensitivity_params : $(SENS_PARAMS)

# --------------------------------------------------
# Prior sensitivity analysis | sensitivity_prior

# Number of prior settings
N_PRIOR_C := 5
N_PRIOR_TAU := 5
PRIOR_C_SETTINGS = $(shell seq $(N_PRIOR_C))
PRIOR_TAU_SETTINGS = $(shell seq $(N_PRIOR_TAU))
# Default setting against which others are compared
PRIOR_BASE_SETTING = 3_3

# Folder structure
PRIOR_SUBDIRS = $(foreach c_set, $(PRIOR_C_SETTINGS),\
								$(foreach tau_set, $(PRIOR_TAU_SETTINGS), $(c_set)_$(tau_set)))
PRIOR_SENS_ROOT = $(addprefix results/,prior_sensitivity)
PRIOR_SENS_DIRS = $(addprefix $(PRIOR_SENS_ROOT)/, $(PRIOR_SUBDIRS))
DIRS += $(PRIOR_SENS_ROOT) $(PRIOR_SENS_DIRS)

# Parametrizations
# Hard coded .csv table
PRIOR_SENS_TABLE_TEMPLATE = templates/prior_sensitivity_settings.csv
PRIOR_SENS_TABLE_FNAME := prior_sensitivity_settings.csv
PRIOR_SENS_TABLE = $(PRIOR_SENS_ROOT)/$(PRIOR_SENS_TABLE_FNAME)

# Alternate prior parametrizations
# Same c_settings with alternative tau settings
# Comparison against the same PRIOR_BASE_DIR
PRIOR_SENS_ALT_ROOT = $(addprefix results/, prior_sensitivity_alt)
PRIOR_SENS_ALT_DIRS = $(addprefix $(PRIOR_SENS_ALT_ROOT)/, $(PRIOR_SUBDIRS))
DIRS += $(PRIOR_SENS_ALT_ROOT) $(PRIOR_SENS_ALT_DIRS)
PRIOR_SENS_ALT_TABLE_TEMPLATE = templates/alt_prior_sensitivity_settings.csv
PRIOR_SENS_ALT_TABLE = $(PRIOR_SENS_ALT_ROOT)/$(PRIOR_SENS_TABLE_FNAME)
PRIOR_SENS_ROOTS = $(PRIOR_SENS_ROOT) $(PRIOR_SENS_ALT_ROOT)
PRIOR_SENS_DIRS_ALL = $(PRIOR_SENS_DIRS) $(PRIOR_SENS_ALT_DIRS)
PRIOR_BASE_DIR = $(PRIOR_SENS_ALT_ROOT)/$(PRIOR_BASE_SETTING)

$(PRIOR_SENS_TABLE) : $(PRIOR_SENS_TABLE_TEMPLATE) | $(PRIOR_SENS_ROOT)
	cp -up $< $@

$(PRIOR_SENS_ALT_TABLE) : $(PRIOR_SENS_ALT_TABLE_TEMPLATE) | $(PRIOR_SENS_ROOT)
	cp -up $< $@

PRIOR_SENS_PARAM_SCRIPT = $(SCRIPT_DIR)/prior_sensitivity_params.R

# Use general parametrization file as template, then set priors for each test setting
PRIOR_SENS_PARAM_TEMPLATE = $(PRIOR_SENS_ROOT)/$(PARAM_FILE_NAME)
PRIOR_SENS_PARAM_TEMPLATES = $(addsuffix /$(PARAM_FILE_NAME), $(PRIOR_SENS_ROOTS))
$(PRIOR_SENS_PARAM_TEMPLATES) : %/$(PARAM_FILE_NAME) : $(PARAM_SCRIPT) $(PARAM_MAIN) | % $(R_SCRIPT)
	$(R_SCRIPT) $(PARAM_FILE_ARGS) -o $(@D)

# Recipes for parameter files are listed later after .SECOND_EXPANSION

sens_prior_ind = $(subst _, , $(lastword $(subst /, , $1)))
PRIOR_SENS_PARAMS = $(addsuffix /$(PARAM_FILE_NAME), $(PRIOR_SENS_DIRS))
PRIOR_SENS_PARAM_FILES = $(addsuffix /$(PARAM_FILE_NAME), $(PRIOR_SENS_DIRS_ALL))

# Model fitting

PRIOR_SENS_MODEL_FILES = $(call dirs_files, $(PRIOR_SENS_DIRS), data.RData model.RData)
PRIOR_SENS_ALT_MODEL_FILES = $(call dirs_files, $(PRIOR_SENS_ALT_DIRS), data.RData model.RData)
.PHONY: prior_sens_fit
prior_sens_fit : $(PRIOR_SENS_MODEL_FILES)

.PHONY: prior_sens_alt_fit
prior_sens_alt_fit: $(PRIOR_SENS_ALT_MODEL_FILES)

# prior sensitivity metrics ------------------------------

# CPO (~ LOO-CV)
CPO_SCRIPT = $(SCRIPT_DIR)/cpo_comparison.R
CPO_FNAME = cpo_comparison.csv

%/$(CPO_FNAME) : $(CPO_SCRIPT) $(addprefix %/, data.RData model.RData) | $(R_SCRIPT)
	$(R_SCRIPT) $(CPO_SCRIPT) -o $@ $(@D)

.PHONY: prior_sens_cpo
prior_sens_cpo : $(addsuffix /$(CPO_FNAME), $(PRIOR_SENS_DIRS))

.PHONY: prior_sens_alt_cpo
prior_sens_alt_cpo : $(addsuffix /$(CPO_FNAME), $(PRIOR_SENS_ALT_DIRS))

# ln intensity surface comparison
SURF_SCRIPT = $(SCRIPT_DIR)/intens_surf_comparison.R
SURF_FNAME = int_surface.csv
%/$(SURF_FNAME) : $(SURF_SCRIPT) $(addprefix %/, data.RData model.RData) $(PRIOR_BASE_DIR)/model.RData | $(R_SCRIPT)
	$(R_SCRIPT) $(SURF_SCRIPT) -o $@ $(PRIOR_BASE_DIR) $(@D)

.PHONY: prior_sens_surf
prior_sens_surf : $(addsuffix /$(SURF_FNAME), $(PRIOR_SENS_DIRS))

.PHONY: prior_sens_alt_surf
prior_sens_alt_surf : $(addsuffix /$(SURF_FNAME), $(PRIOR_SENS_ALT_DIRS))


# skill coefficient comparison
PRIOR_SKILL_SCRIPT = $(SCRIPT_DIR)/prior_sens_skill_comparison.R
PRIOR_SKILL_FNAME = skill_comparison.csv
PRIOR_SKILL_COMPS = $(addsuffix /$(PRIOR_SKILL_FNAME), $(PRIOR_SENS_DIRS))
PRIOR_SKILL_SUM_FNAME = skill_comparison_summary.csv
PRIOR_SKILL_SUM = $(PRIOR_SENS_ROOT)/$(PRIOR_SKILL_SUM_FNAME)
PRIOR_SKILL_SUMS = $(addsuffix $(PRIOR_SKILL_SUM_FNAME), $(PRIOR_SENS_DIRS_ALL))

%/$(PRIOR_SKILL_FNAME) : $(PRIOR_SKILL_SCRIPT) $(addprefix %/, data.RData model.RData) | $(R_SCRIPT)
	$(R_SCRIPT) $(PRIOR_SKILL_SCRIPT)  -o $@ $(@D)

%/$(PRIOR_SKILL_SUM_FNAME) : $(addprefix %/, $(addsuffix /$(PRIOR_SKILL_FNAME), $(PRIOR_SUBDIRS))) | $(CSVSTACK)
	$(CSVSTACK) $^ > $@

.PHONY: prior_sens_comparison
prior_sens_comparison : $(call dirs_files, $(PRIOR_SENS_DIRS),\
	$(POST_SAMP_FNAME) comparison.csv)

.PHONY: prior_sens_analysis
prior_sens_analysis: prior_sens_param_files prior_sens_fit

.PHONY: prior_sens_alt_analysis
prior_sens_alt_analysis: prior_sens_alt_param_files prior_sens_alt_fit

.PHONY: prior_sens_skill prior_sens_alt_skill
prior_sens_skill: $(PRIOR_SENS_ROOT)/$(PRIOR_SKILL_SUM_FNAME)

prior_sens_alt_skill: $(PRIOR_SENS_ALT_ROOT)/$(PRIOR_SKILL_SUM_FNAME)

# Separate target for intermediate results so Make doesn't delete them
.PHONY: prior_sens_alt_skill_int
prior_sens_alt_skill_int: $(addprefix $(PRIOR_SENS_ALT_ROOT)/, $(addsuffix /$(PRIOR_SKILL_FNAME), $(PRIOR_SUBDIRS)))

#-----------------------------------------------
# Manuscript figures  | manuscript_figures
#-----------------------------------------------
# Figures appearing in a manuscript or its supplement

FIGURE_DIR := figures
DIRS += $(FIGURE_DIR)

# Model comparison figures
MOD_COMP_FIG_DIR = $(FIGURE_DIR)/model_comparison
DIRS += $(MOD_COMP_FIG_DIR)

# Main model highlighted in manuscript with some extra figures
MAIN_MODEL ?= nb_beta
MAIN_MOD_DIR := $(filter %/$(MAIN_MODEL),$(RESULT_DIRS))
MAIN_MOD_FIGS = $(call dirs_files, $(MAIN_MOD_DIR), $(MAP_FIGS))

# Copy figures from individual results folders to FIGURE_DIR for convenience
# Figures to copy
MOD_COMP_SRC_FIGS = $(PORA_FIG_PATHS) $(MAIN_MOD_FIGS)
MOD_COMP_SRC_DIRS = $(dir $(MOD_COMP_SRC_FIGS))
MOD_COMP_SRC_PARENTS = $(foreach fig_path, $(MOD_COMP_SRC_FIGS), $(call parent_name, $(fig_path)))
# Destination paths
MOD_COMP_DEST_DIRS = $(addprefix $(MOD_COMP_FIG_DIR)/, $(MOD_COMP_SRC_PARENTS))
MOD_COMP_DEST_FIGS = $(join $(addsuffix /,$(MOD_COMP_DEST_DIRS)), $(notdir $(MOD_COMP_SRC_FIGS)))

.PHONY : model_comp_figures
model_comp_figures : $(MOD_COMP_DEST_FIGS)

# Second expansion required by some recipes,
# due to filter and pattern rules both using '%' for different things, see e.g.:
# https://stackoverflow.com/questions/11238052/can-prerequisites-in-a-static-pattern-rule-be-filtered
# https://stackoverflow.com/questions/71848591/how-can-i-pass-a-rule-pattern-in-a-makefile-to-filter-dependencies
# here $* is the expanded rule pattern stem and % is the filter pattern
get_src_fig = $(filter %/$*, $(MOD_COMP_SRC_FIGS))

.SECONDEXPANSION:
$(MOD_COMP_DEST_FIGS): $(MOD_COMP_FIG_DIR)/% : $$(get_src_fig)
	cp -up $< $(@D)

# -----------------------------------------------------------------------------
# Recipes for cross-validation that require SECONDEXPANSION due to hiarchical folder structure

# perm_1/folds.csv perm_2/folds.csv ...
get_index_files = $(filter $*/%, $(CV_IND) )
get_fold_csv = $(dir $*)folds.csv

# Create index file for a each cross-validation fold in a given permutation
# perm_1/fold_1/ind_train.txt perm_1/fold_2/ind_train.txt  ... perm_2/fold_1/ind_train.txt ...
%/ind_train.txt : $$(get_fold_csv) | % $(IND_SCRIPT)
	$(R_SCRIPT) $(IND_SCRIPT) -n $(call fold_num, $(@D)) -o $(@D) $(call perm_file, $(@D))

# Divide data into training and test set based on the indices in the current
# cross-validation permutation and fold.
# Assume index files haven't changed (when they already exist)
# to avoid rerunning old cross-validation permutation when running additional ones
%/data.RData : $(CNSTR_DATA_DEP) %/parameters.RData | $$(get_index_files) $(R_SCRIPT)
	$(R_SCRIPT) $(CNSTR_DATA_SCRIPT) $(@D) $(@D)/parameters.RData

# --------------------------------------------------
# Prior sensitivity analysis
get_dir = $(dir $*)
$(PRIOR_SENS_PARAM_FILES) : %/$(PARAM_FILE_NAME) : $$(get_dir)$(PARAM_FILE_NAME) $$(get_dir)$(PRIOR_SENS_TABLE_FNAME) $(PRIOR_SENS_PARAM_SCRIPT) | %
	$(R_SCRIPT) $(PRIOR_SENS_PARAM_SCRIPT) -i $< -o $(@D) $(word 2, $^) $(call sens_prior_ind, $(@D))

.PHONY: prior_sens_param_files
prior_sens_param_files : $(PRIOR_SENS_PARAMS)

.PHONY: prior_sens_alt_param_files
prior_sens_alt_param_files : $(addsuffix /$(PARAM_FILE_NAME), $(PRIOR_SENS_ALT_DIRS))

# combine cpo and intensity surface results
SENS_COMB_SCRIPT = $(SCRIPT_DIR)/prior_sens_summary.R
SENS_COMB_SUM_FNAME = prior_sens_summary.csv
SENS_COMB_SUMS = $(addsuffix /$(SENS_COMB_SUM_FNAME), $(PRIOR_SENS_DIRS))
SENS_SUM_FNAME = prior_sens_comparison_summary.csv
SENS_SUMMARY = $(PRIOR_SENS_ROOT)/$(SENS_SUM_FNAME)

%/$(SENS_COMB_SUM_FNAME) : $(SENS_COMB_SCRIPT) $(addprefix %/, $(SURF_FNAME) $(CPO_FNAME)) $$(get_dir)$(PRIOR_SENS_TABLE_FNAME)  | $(R_SCRIPT)
		$(R_SCRIPT) $< -o $@ $(wordlist 2, $(words $^), $^)

%/prior_sens_comparison_summary.csv : $(addprefix %/, $(addsuffix /$(SENS_COMB_SUM_FNAME), $(PRIOR_SUBDIRS))) | $(CSVSTACK)
	$(CSVSTACK) $^ > $@

.PHONY: prior_sens_sums
prior_sens_sums: $(SENS_COMB_SUMS) $(SENS_SUMMARY)

.PHONY: prior_sens_alt_sums
prior_sens_alt_sums: $(addprefix $(PRIOR_SENS_ALT_ROOT)/, $(addsuffix /$(SENS_COMB_SUM_FNAME), $(PRIOR_SUBDIRS)))

prior_sens_alt_sums: $(PRIOR_SENS_ALT_ROOT)/$(SENS_SUM_FNAME)



# ------------------------------------------------------------
# Miscellaneous figures in supplement | supplement_figures

# Sensitivity analysis comparison
CV_COMP_FIG_SCRIPT = $(SCRIPT_DIR)/sensitivity_comparison_plot.R
CV_COMP_FIG = $(SENS_DIR)/sensitivity_comparison.png

.PHONY: cv_comp_fig
cv_comp_fig : $(CV_COMP_FIG)

$(CV_COMP_FIG) : $(CV_COMP_RAW_TB) $(CV_COMP_FIG_SCRIPT)  | $(R_SCRIPT)
	$(R_SCRIPT) $(CV_COMP_FIG_SCRIPT) -o $@ $<

# Log intensity vs logit presence comparison
INTENS_FIG_1 = $(FIGURE_DIR)/intensity_vs_pres.png
INTENS_FIG_2 = $(FIGURE_DIR)/intensity_vs_pres_diff.png
INTENS_VS_PRES_SCRIPT = $(SCRIPT_DIR)/intensity_to_presence.R

# Binomial approximation for expert likelihoods
BIN_APPROX_FIGS = $(addprefix $(FIGURE_DIR)/, beta_bin_categories_cdf.png beta_bin_categories_cdf_logit.png)
BIN_APPROX_SCRIPT = $(SCRIPT_DIR)/beta_bin_figures.R
BIN_APPROX_DEPS = $(SRC_DIR)/beta_hyperpar_fit.R

$(BIN_APPROX_FIGS) : $(BIN_APPROX_SCRIPT) $(BIN_APPROX_DEPS) | $(R_SCRIPT)
	$(R_SCRIPT) $(BIN_APPROX_SCRIPT) -o $(FIGURE_DIR)

# Maps for observations and random effect meshes
SUPP_MAP_SCRIPT = $(SCRIPT_DIR)/supplement_maps.R
OBS_FIG = $(FIGURE_DIR)/observations_map.png
BARRIER_MESH_FIG = $(FIGURE_DIR)/barrier_mesh.png
EXPERT_MESH_FIG = $(FIGURE_DIR)/expert_mesh.png
MESH_FIGS = $(BARRIER_MESH_FIG) $(EXPERT_MESH_FIG)

# Data used as basis for plotting obsrvation figure
BASE_RESULT_DIR = $(firstword $(RESULT_DIRS))
BASE_FILES = $(addprefix $(BASE_RESULT_DIR)/, data.RData model.RData parameters.RData)

$(OBS_FIG) : $(BASE_FILES) $(SUPP_MAP_SCRIPT) $(SKILL_RANK_REFERENCE) | $(R_SCRIPT)
	$(R_SCRIPT) $(SUPP_MAP_SCRIPT) $(BASE_RESULT_DIR) $(@D) 'observations' $(SKILL_RANK_REFERENCE)

$(BARRIER_MESH_FIG) : $(BASE_FILES) $(SUPP_MAP_SCRIPT) | $(R_SCRIPT)
	$(R_SCRIPT) $(SUPP_MAP_SCRIPT) $(BASE_RESULT_DIR) $(@D) 'mesh'

$(EXPERT_MESH_FIG) : $(BARRIER_MESH_FIG)
	$(call update_stamp, $@)

.PHONY : supplement

supplement : $(INTENS_FIG_1) $(INTENS_FIG_2) $(OBS_FIG) $(MESH_FIGS) $(BIN_APPROX_FIGS)

# Same script produces both figures
$(INTENS_FIG_2) : $(INTENS_VS_PRES_SCRIPT) | $(RENV_STAMP) $(FIGURE_DIR)
	 $(R_SCRIPT) $(INTENS_VS_PRES_SCRIPT) $(FIGURE_DIR)

$(INTENS_FIG_1) : $(INTENS_FIG_2)
	$(call update_stamp, $@)

# Additional figures for distribution pdfs
# --------------------------------------------------
#
SENS_PDF_SCRIPT = $(SCRIPT_DIR)/plot_sensitivity_pdfs.R
SENS_PDF_FIG = $(addprefix $(FIGURE_DIR)/,\
							 prior_sensitivity_c_sigma.png prior_sensitivity_tau_bym.png)

$(firstword $(SENS_PDF_FIG)) : $(SENS_PDF_SCRIPT) | $(R_SCRIPT) $(FIGURE_DIR)
	$(R_SCRIPT) $(SENS_PDF_SCRIPT) -o $(@D)

$(call other_words, $(SENS_PDF_FIG)) : $(firstword $(SENS_PDF_FIG))
	$(call update_stamp, $@)

.PHONY: pdf_figures
pdf_figures : $(SENS_PDF_FIG)

PRIOR_SKILL_FIG_SCRIPT = $(SCRIPT_DIR)/prior_sens_skill_figure.R
PRIOR_SENS_FIG_DIR = $(FIGURE_DIR)/prior_sensitivity
DIRS += $(PRIOR_SENS_FIG_DIR)
PRIOR_SKILL_FIG_FNAME =  prior_sens_skill_comparison.png
PRIOR_SKILL_FIG = $(FIGURE_DIR)/$(PRIOR_SKILL_FIG_FNAME)

PRIOR_SKILL_FIGS = $(addsuffix /$(PRIOR_SKILL_FIG_FNAME), $(PRIOR_SENS_ROOTS))

$(PRIOR_SKILL_FIGS) : %/$(PRIOR_SKILL_FIG_FNAME) : \
	$(PRIOR_SKILL_FIG_SCRIPT) %/$(PRIOR_SKILL_SUM_FNAME) %/$(PRIOR_SENS_TABLE_FNAME) | $(R_SCRIPT)
	$(R_SCRIPT) $^ -o $@

prior_sens_skill: $(PRIOR_SENS_ROOT)/$(PRIOR_SKILL_FIG_FNAME)

prior_sens_alt_skill: $(PRIOR_SENS_ALT_ROOT)/$(PRIOR_SKILL_FIG_FNAME)

PRIOR_SENS_UTIL_FCNS = $(SRC_DIR)/prior_sensitivity_util.R
PRIOR_SENS_COMP_FIGS = prior_sens_lpd.png prior_sens_surf_diff.png prior_sens_surf_var.png

PRIOR_SENS_COMP_FIG_SCRIPT = $(SCRIPT_DIR)/prior_sens_comp_figure.R
.PHONY: prior_sens_comp_figs prior_sens_alt_comp_figs
prior_sens_comp_figs: $(addprefix $(PRIOR_SENS_ROOT)/, $(PRIOR_SENS_COMP_FIGS))

prior_sens_alt_comp_figs: $(addprefix $(PRIOR_SENS_ALT_ROOT)/, $(PRIOR_SENS_COMP_FIGS))

%/prior_sens_lpd.png : $(PRIOR_SENS_COMP_FIG_SCRIPT) %/$(SENS_SUM_FNAME) %/$(PRIOR_SENS_TABLE_FNAME) | $(R_SCRIPT)
	$(R_SCRIPT) $^ -o $(@D)

%/prior_sens_surf_diff.png :  %/prior_sens_lpd.png
	$(call update_stamp, $@)

%/prior_sens_surf_var.png :  %/prior_sens_lpd.png
	$(call update_stamp, $@)

# --------------------------------------------------------------------

#---------------------------------------------------------------------
# Create folders | mk_dirs
# Use pipe '|' before these to mark them as order-only dependencies.
# This prevents them from being endlessly remade every time something inside the folder changes.

.PHONY: dirs

dirs: $(DIRS)

# Creating folders even when -t (touch) flag is set
# This flag can be used to update time stamps for targets without running the recipe.
$(DIRS) :
ifneq (,$(findstring t,$(firstword -$(MAKEFLAGS))))
	+mkdir -p $@
else
	mkdir -p $@
endif


#------------------------------------------------------------
# Debugging targets
#------------------------------------------------------------
.PHONY: variables
variables:
	$(info "PROJECT_ROOT": $(PROJECT_ROOT))
	$(info "BIN": $(BIN))
	$(info "PY_INSTALL_STAMP": $(PY_INSTALL_STAMP))
	$(info "LOCAL_RDIR": $(LOCAL_RDIR))
	$(info "R_PREFIX": $(R_PREFIX))
	$(info "R_BASE_STAMP": $(R_BASE_STAMP))
	$(info "DATA_FILES": $(DATA_FILES))
	$(info "DATA_TMP_FILES": $(DATA_TMP_FILES))
	$(info "COV_RAST_FILE": $(COV_RAST_FILE))
	$(info "RENV_STAMP": $(RENV_STAMP))
	$(info "PATH": $(PATH))
	$(info "R_BINARIES": $(R_BINARIES))
	$(info "R_EXEC": $(R_EXEC))
	$(info "COMPILE_R": $(COMPILE_R))
	$(info "RESULTS_DIR": $(RESULTS_DIR))
	$(info "RESULT_DIRS": $(RESULT_DIRS))
	$(info "FIG_PATHS:" $(FIG_PATHS))
	$(info "MAKEFLAGS:" $(MAKEFLAGS))

.PHONY: print_dirs
print_dirs:
	$(info $(DIRS))