Add Scratch.jl

Project.toml

@@ -9,6 +9,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
 
@@ -20,9 +21,24 @@ RasterDataSourcesProjExt = "Proj"
 
 [compat]
 ASCIIrasters = "0.1"
+Dates = "1"
 HTTP = "0.8, 0.9, 1"
 JSON = "0.21"
 Proj = "1"
+Pkg = "1"
+SafeTestsets = "0.0.1"
+Scratch = "1"
 URIs = "1"
 ZipFile = "0.9, 0.10"
-julia = "1.9"
+julia = "1.9"
+
+[extras]
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Proj = "c94c279d-25a6-4763-9509-64d165bea63e"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Aqua", "Dates", "Pkg", "Proj", "SafeTestsets", "Test"]

README.md

@@ -57,14 +57,24 @@ Install as usual with:
 ] add RasterDataSources
 ```
 
-To download data you will need to specify a folder to put it in. You can do this
-by assigning the environment variable `RASTERDATASOURCES_PATH`:
+### Storage Configuration
+
+RasterDataSources.jl handles data storage for you. By default, it will create a 
+persistent scratch directory to store downloaded raster data.
+However, raster data may be 100s of GB, or more. So make sure there is room in your home directory. 
+
+#### Custom Storage Location
+
+To put data in a custom location, set `RASTERDATASOURCES_PATH` in your
+environment, usually in your `[juliadir]/config/startup.jl` file:
 
 ```julia
-ENV["RASTERDATASOURCES_PATH"] = "/home/user/Data/"
+ENV["RASTERDATASOURCES_PATH"] = "/path/to/your/data"
 ```
 
-This can be put in your `startup.jl` file or the system environment.
+#### Finding Your Storage Location
+
+`RasterDataSources.rasterpath()` will return the current data storage path.
 
 
-RasterDataSources was based on code from the `SimpleSDMDataSoures.jl` package by Timothée Poisot.
+_RasterDataSources was originally based on code from the `SimpleSDMDataSoures.jl` package by Timothée Poisot._

src/RasterDataSources.jl

@@ -10,7 +10,8 @@ using Dates,
       URIs,
       ZipFile,
       ASCIIrasters,
-      DelimitedFiles
+      DelimitedFiles,
+      Scratch
 
 import JSON.Parser as JP
 

src/alwb/alwb.jl

@@ -67,7 +67,7 @@ This will return the file containing annual averages, including your date:
 
 ```julia
 julia> getraster(ALWB{Values,Year}, :ss_pct; date=Date(2001, 2))
-"/your/RASTERDATASOURCES_PATH/ALWB/values/month/ss_pct.nc"
+"/path/to/storage/ALWB/values/month/ss_pct.nc"
 ```
 
 Returns the filepath/s of the downloaded or pre-existing files.

src/modis/products.jl

@@ -17,7 +17,7 @@ end
 """
     Lists available layers for a given MODIS Product
 
-Looks in `joinpath(ENV["RASTERDATASOURCES_PATH"]/MODIS/layers` for
+Looks in the storage directory under `MODIS/layers` for
 a file with the right name. If not found, sends a request to the server
 to get the list.
 

src/shared.jl

@@ -1,3 +1,6 @@
+const RDS_PATH_STRING = "ENV[\"RASTERDATASOURCES_PATH\"] = \"/path/to/your/data\""
+const STARTUP_PATH_STRING = "`[julia folder]/config/startup.jl`"
+
 # Vector layers are allowed, but converted to `Tuple` immediatedly.
 function getraster(T::Type, layers::AbstractArray; kw...)
     getraster(T, (layers...,); kw...)
@@ -62,6 +65,7 @@ function _maybe_download(uri::URI, filepath, headers = [])
     if !isfile(filepath)
         mkpath(dirname(filepath))
         @info "Starting download for $uri"
+
         try
             HTTP.download(string(uri), filepath, headers)
         catch e
@@ -73,11 +77,58 @@ function _maybe_download(uri::URI, filepath, headers = [])
     filepath
 end
 
+"""
+    rasterpath()
+
+Returns the absolute path to the directory where raster data is stored.
+
+The storage location is determined using the following priority order:
+1. If the `RASTERDATASOURCES_PATH` environment variable is set and points to a valid directory, that path is used
+2. If no environment variable is set, a persistent scratch directory is automatically created using Scratch.jl
+
+The scratch directory persists across Julia sessions and package updates, ensuring downloaded data is not lost.
+If scratch directory creation fails, an error is thrown with instructions to manually set the environment variable.
+
+# Examples
+```julia
+# With environment variable set
+$RDS_PATH_STRING
+rasterpath()  # Returns "/path/to/your/data"
+
+# Without environment variable (automatic scratch directory)
+rasterpath()  # Returns something like "/Users/username/.julia/scratchspaces/12345.../raster_data"
+```
+"""
 function rasterpath()
-    if haskey(ENV, "RASTERDATASOURCES_PATH") && isdir(ENV["RASTERDATASOURCES_PATH"])
-        ENV["RASTERDATASOURCES_PATH"]
-    else
-        error("You must set `ENV[\"RASTERDATASOURCES_PATH\"]` to a path in your system")
+    # Priority 1: Use environment variable if set and valid
+    if haskey(ENV, "RASTERDATASOURCES_PATH")
+        path = ENV["RASTERDATASOURCES_PATH"]
+        isdir(path) || error("Your RASTERDATASOURCES_PATH is not a directory: $path")
+        return path
+    end
+
+    # Priority 2: Use scratch directory
+    try
+        scratch_dir = @get_scratch!("raster_data")
+        if isempty(readdir(scratch_dir)) # If this is the first use, print info in the REPL
+            @info """
+                Created scratch directory for raster data storage: 
+                $scratch_dir. 
+                Make sure there is adequate space, some datasets are 100GB+. 
+                For a custom location set $RDS_PATH_STRING 
+                in your $STARTUP_PATH_STRING file."
+                """ maxlog=1
+        end
+        return scratch_dir
+    catch e
+        error(
+            """
+            Failed to create scratch directory for raster data storage.
+            Please set $RDS_PATH_STRING manually 
+            in your $STARTUP_PATH_STRING.
+            Error: $e
+            """
+        )
     end
 end
 

test/awap.jl

@@ -4,14 +4,14 @@ using RasterDataSources: rastername, rasterpath, zipurl, zipname, zippath
 @testset "AWAP" begin
     using RasterDataSources: rastername, zipurl, zipname, zippath
 
-    raster_file = joinpath(ENV["RASTERDATASOURCES_PATH"], "AWAP", "vprp", "vprph09", "20010101.grid")
+    raster_file = joinpath(rasterpath(), "AWAP", "vprp", "vprph09", "20010101.grid")
     @test rasterpath(AWAP, :vprpress09; date=Date(2001, 1)) == raster_file
     @test rastername(AWAP, :vprpress09; date=Date(2001, 1)) == "20010101.grid" 
 
     @test zipurl(AWAP, :vprpress09; date=Date(2001, 1)) ==
         URI(scheme="http", host="www.bom.gov.au", path="/web03/ncc/www/awap/vprp/vprph09/daily/grid/0.05/history/nat/2001010120010101.grid.Z")
     @test zippath(AWAP, :vprpress09; date=Date(2001, 1)) ==
-        joinpath(ENV["RASTERDATASOURCES_PATH"], "AWAP", "vprp", "vprph09", "20010101.grid.Z")
+        joinpath(rasterpath(), "AWAP", "vprp", "vprph09", "20010101.grid.Z")
     @test zipname(AWAP, :vprpress09; date=Date(2001, 1)) == "20010101.grid.Z"
 
     if Sys.islinux()

test/runtests.jl

@@ -30,3 +30,4 @@ end
 # @time @safetestset "modis utilities" begin include("modis-utilities.jl") end
 @time @safetestset "modis product info" begin include("modis-products.jl") end
 # @time @safetestset "modis interface" begin include("modis-interface.jl") end
+@time @safetestset "storage path resolution" begin include("storage-path.jl") end