From f3207059ce9bd5eda554778897a3bdb6c2ce429f Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 08:42:13 -0400 Subject: [PATCH 01/14] Allow compress to be a character --- R/driver_rds.R | 25 +++++++++++++++++++++---- R/hash.R | 2 +- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index 5a30f59..245a53e 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -55,7 +55,12 @@ ##' might be nice for persistent application data. ##' ##' @param compress Compress the generated file? This saves a small -##' amount of space for a reasonable amount of time. +##' amount of space for a reasonable amount of time. Possible values +##' are \code{"none"} for no compression, \code{"gzfile"} to write +##' to a \code{gzfile} connection (default), and \code{"fst"} to use +##' \code{compress_fst} (recommended). To preserve back compatibility, +##' \code{compress} can also be logical: \code{TRUE} for \code{"gzfile"} +##' and \code{FALSE} for \code{"none"}. ##' ##' @param mangle_key Mangle keys? If TRUE, then the key is encoded ##' using base64 before saving to the filesystem. See Details. @@ -76,6 +81,7 @@ ##' ##' @param default_namespace Default namespace (see ##' \code{\link{storr}}). +##' ##' @export ##' @examples ##' @@ -154,7 +160,7 @@ R6_driver_rds <- R6::R6Class( ## future versions of driver_rds can use to patch, warn or ## change behaviour with older versions of the storr. if (!is_new && !file.exists(driver_rds_config_file(path, "version"))) { - write_if_missing("1.0.1", driver_rds_config_file(path, "version")) + write_if_missing("1.2.2", driver_rds_config_file(path, "version")) write_if_missing("TRUE", driver_rds_config_file(path, "mangle_key_pad")) write_if_missing("TRUE", driver_rds_config_file(path, "compress")) write_if_missing("md5", driver_rds_config_file(path, "hash_algorithm")) @@ -177,10 +183,11 @@ R6_driver_rds <- R6::R6Class( FALSE, TRUE) if (!is.null(compress)) { - assert_scalar_logical(compress) + compress <- as.character(compress) } self$compress <- driver_rds_config(path, "compress", compress, - TRUE, FALSE) + "TRUE", FALSE) + self$compress <- parse_rds_compress(self$compress) if (!is.null(hash_algorithm)) { assert_scalar_character(hash_algorithm) @@ -452,3 +459,13 @@ See 'Corrupt keys' within ?storr_rds for how to proceed" -> fmt message(sprintf(fmt, length(files), namespace, path, files)) corrupt_notices[[path]] <- now } + +parse_rds_compress <- function(compress) { + if (identical(compress, "TRUE")) { + "gzfile" + } else if (identical(compress, "FALSE")) { + "none" + } else { + compress + } +} diff --git a/R/hash.R b/R/hash.R index 80fb64e..1af0865 100644 --- a/R/hash.R +++ b/R/hash.R @@ -87,7 +87,7 @@ try_write_serialized_rds <- function(value, filename, compress, scratch_dir = NULL, long = 2^31 - 2) { tmp <- tempfile(tmpdir = scratch_dir %||% tempdir()) - con <- (if (compress) gzfile else file)(tmp, "wb") + con <- (if (identical(compress, "gzfile")) gzfile else file)(tmp, "wb") needs_close <- TRUE on.exit(if (needs_close) close(con), add = TRUE) len <- length(value) From cde64909d8b6a6f6136b31f88db10c7f13cf53fb Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 08:53:48 -0400 Subject: [PATCH 02/14] Add tests of compression back compatibility --- tests/testthat/test-driver-rds.R | 27 ++++++++++++++++++ .../v1.2.1_compress_FALSE/config/compress | 1 + .../config/hash_algorithm | 1 + .../v1.2.1_compress_FALSE/config/mangle_key | 1 + .../config/mangle_key_pad | 1 + .../v1.2.1_compress_FALSE/config/version | 1 + .../data/6abd4188389195dfdfa9a37665e4cca6.rds | Bin 0 -> 35 bytes .../v1.2.1_compress_FALSE/keys/objects/key | 1 + .../v1.2.1_compress_TRUE/config/compress | 1 + .../config/hash_algorithm | 1 + .../v1.2.1_compress_TRUE/config/mangle_key | 1 + .../config/mangle_key_pad | 1 + .../v1.2.1_compress_TRUE/config/version | 1 + .../data/6abd4188389195dfdfa9a37665e4cca6.rds | Bin 0 -> 49 bytes .../v1.2.1_compress_TRUE/keys/objects/key | 1 + 15 files changed, 39 insertions(+) create mode 100644 tests/testthat/v1.2.1_compress_FALSE/config/compress create mode 100644 tests/testthat/v1.2.1_compress_FALSE/config/hash_algorithm create mode 100644 tests/testthat/v1.2.1_compress_FALSE/config/mangle_key create mode 100644 tests/testthat/v1.2.1_compress_FALSE/config/mangle_key_pad create mode 100644 tests/testthat/v1.2.1_compress_FALSE/config/version create mode 100644 tests/testthat/v1.2.1_compress_FALSE/data/6abd4188389195dfdfa9a37665e4cca6.rds create mode 100644 tests/testthat/v1.2.1_compress_FALSE/keys/objects/key create mode 100644 tests/testthat/v1.2.1_compress_TRUE/config/compress create mode 100644 tests/testthat/v1.2.1_compress_TRUE/config/hash_algorithm create mode 100644 tests/testthat/v1.2.1_compress_TRUE/config/mangle_key create mode 100644 tests/testthat/v1.2.1_compress_TRUE/config/mangle_key_pad create mode 100644 tests/testthat/v1.2.1_compress_TRUE/config/version create mode 100644 tests/testthat/v1.2.1_compress_TRUE/data/6abd4188389195dfdfa9a37665e4cca6.rds create mode 100644 tests/testthat/v1.2.1_compress_TRUE/keys/objects/key diff --git a/tests/testthat/test-driver-rds.R b/tests/testthat/test-driver-rds.R index 01dbcec..bdd5f4d 100644 --- a/tests/testthat/test-driver-rds.R +++ b/tests/testthat/test-driver-rds.R @@ -176,6 +176,33 @@ test_that("backward compatibility", { "Incompatible value for hash_algorithm") }) +test_that("backward compatibility: compression", { + ## In version 1.2.1 and earlier, the compress argument was a logical scalar. + path <- copy_to_tmp("v1.2.1_compress_TRUE") + st <- storr_rds(path) + expect_equal(st$list(), "key") + expect_equal(st$get("key"), "value") + for (compress in list("gzfile", TRUE)) { + expect_silent(st <- storr_rds(path, compress = compress)) + } + for (compress in list("none", "fst", FALSE)) { + expect_error(st <- storr_rds(path, compress = compress), + "Incompatible value for compress") + } + + path <- copy_to_tmp("v1.2.1_compress_FALSE") + st <- storr_rds(path) + expect_equal(st$list(), "key") + expect_equal(st$get("key"), "value") + for (compress in list("none", FALSE)) { + expect_silent(st <- storr_rds(path, compress = compress)) + } + for (compress in list("gzfile", "fst", TRUE)) { + expect_error(st <- storr_rds(path, compress = compress), + "Incompatible value for compress") + } +}) + test_that("mangledness padding backward compatibility", { ## In version 1.0.1 and earlier, mangling was always padded path <- copy_to_tmp("v1.0.1_mangled") diff --git a/tests/testthat/v1.2.1_compress_FALSE/config/compress b/tests/testthat/v1.2.1_compress_FALSE/config/compress new file mode 100644 index 0000000..f6d449c --- /dev/null +++ b/tests/testthat/v1.2.1_compress_FALSE/config/compress @@ -0,0 +1 @@ +FALSE diff --git a/tests/testthat/v1.2.1_compress_FALSE/config/hash_algorithm b/tests/testthat/v1.2.1_compress_FALSE/config/hash_algorithm new file mode 100644 index 0000000..9d39e6b --- /dev/null +++ b/tests/testthat/v1.2.1_compress_FALSE/config/hash_algorithm @@ -0,0 +1 @@ +md5 diff --git a/tests/testthat/v1.2.1_compress_FALSE/config/mangle_key b/tests/testthat/v1.2.1_compress_FALSE/config/mangle_key new file mode 100644 index 0000000..f6d449c --- /dev/null +++ b/tests/testthat/v1.2.1_compress_FALSE/config/mangle_key @@ -0,0 +1 @@ +FALSE diff --git a/tests/testthat/v1.2.1_compress_FALSE/config/mangle_key_pad b/tests/testthat/v1.2.1_compress_FALSE/config/mangle_key_pad new file mode 100644 index 0000000..f6d449c --- /dev/null +++ b/tests/testthat/v1.2.1_compress_FALSE/config/mangle_key_pad @@ -0,0 +1 @@ +FALSE diff --git a/tests/testthat/v1.2.1_compress_FALSE/config/version b/tests/testthat/v1.2.1_compress_FALSE/config/version new file mode 100644 index 0000000..6085e94 --- /dev/null +++ b/tests/testthat/v1.2.1_compress_FALSE/config/version @@ -0,0 +1 @@ +1.2.1 diff --git a/tests/testthat/v1.2.1_compress_FALSE/data/6abd4188389195dfdfa9a37665e4cca6.rds b/tests/testthat/v1.2.1_compress_FALSE/data/6abd4188389195dfdfa9a37665e4cca6.rds new file mode 100644 index 0000000000000000000000000000000000000000..8f47868e9e60408bde7ec47c26d1a889c55ff41f GIT binary patch literal 35 kcma#xVqjokVqj(ilFUFRg8&dSGO#dk0%_K=#GKMp02g5bvj6}9 literal 0 HcmV?d00001 diff --git a/tests/testthat/v1.2.1_compress_FALSE/keys/objects/key b/tests/testthat/v1.2.1_compress_FALSE/keys/objects/key new file mode 100644 index 0000000..9b41691 --- /dev/null +++ b/tests/testthat/v1.2.1_compress_FALSE/keys/objects/key @@ -0,0 +1 @@ +6abd4188389195dfdfa9a37665e4cca6 diff --git a/tests/testthat/v1.2.1_compress_TRUE/config/compress b/tests/testthat/v1.2.1_compress_TRUE/config/compress new file mode 100644 index 0000000..ef2f513 --- /dev/null +++ b/tests/testthat/v1.2.1_compress_TRUE/config/compress @@ -0,0 +1 @@ +TRUE diff --git a/tests/testthat/v1.2.1_compress_TRUE/config/hash_algorithm b/tests/testthat/v1.2.1_compress_TRUE/config/hash_algorithm new file mode 100644 index 0000000..9d39e6b --- /dev/null +++ b/tests/testthat/v1.2.1_compress_TRUE/config/hash_algorithm @@ -0,0 +1 @@ +md5 diff --git a/tests/testthat/v1.2.1_compress_TRUE/config/mangle_key b/tests/testthat/v1.2.1_compress_TRUE/config/mangle_key new file mode 100644 index 0000000..f6d449c --- /dev/null +++ b/tests/testthat/v1.2.1_compress_TRUE/config/mangle_key @@ -0,0 +1 @@ +FALSE diff --git a/tests/testthat/v1.2.1_compress_TRUE/config/mangle_key_pad b/tests/testthat/v1.2.1_compress_TRUE/config/mangle_key_pad new file mode 100644 index 0000000..f6d449c --- /dev/null +++ b/tests/testthat/v1.2.1_compress_TRUE/config/mangle_key_pad @@ -0,0 +1 @@ +FALSE diff --git a/tests/testthat/v1.2.1_compress_TRUE/config/version b/tests/testthat/v1.2.1_compress_TRUE/config/version new file mode 100644 index 0000000..6085e94 --- /dev/null +++ b/tests/testthat/v1.2.1_compress_TRUE/config/version @@ -0,0 +1 @@ +1.2.1 diff --git a/tests/testthat/v1.2.1_compress_TRUE/data/6abd4188389195dfdfa9a37665e4cca6.rds b/tests/testthat/v1.2.1_compress_TRUE/data/6abd4188389195dfdfa9a37665e4cca6.rds new file mode 100644 index 0000000000000000000000000000000000000000..9e05a974c0d0d2ff1a62b971a103ccdddb079e06 GIT binary patch literal 49 zcmb2|=3oE==I#ec2?+^F32Dre&N!$wGYA+PZ7?Wc%DlGe$ec?d3|vL*1 Date: Tue, 18 Jun 2019 09:03:27 -0400 Subject: [PATCH 03/14] Test back-compatible compression API --- R/driver_rds.R | 12 ++++++++---- man/storr.Rd | 2 +- man/storr_rds.Rd | 11 ++++++++++- tests/testthat/test-driver-rds.R | 18 +++++++----------- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index 245a53e..18259f8 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -58,9 +58,13 @@ ##' amount of space for a reasonable amount of time. Possible values ##' are \code{"none"} for no compression, \code{"gzfile"} to write ##' to a \code{gzfile} connection (default), and \code{"fst"} to use -##' \code{compress_fst} (recommended). To preserve back compatibility, -##' \code{compress} can also be logical: \code{TRUE} for \code{"gzfile"} -##' and \code{FALSE} for \code{"none"}. +##' \code{compress_fst()} (recommended, but requires the \code{fst} package). +##' To preserve back compatibility, \code{compress} can also be logical: +##' \code{TRUE} for \code{"gzfile"} and \code{FALSE} for \code{"none"}. +##' However, these values are not interchangeable for existings \code{storr}s. +##' For example, if you create a \code{storr} with \code{compress = TRUE}, +##' you must continue to use \code{compress = TRUE} +##' and not \code{compress = "gzfile"} when you recover it later. ##' ##' @param mangle_key Mangle keys? If TRUE, then the key is encoded ##' using base64 before saving to the filesystem. See Details. @@ -186,7 +190,7 @@ R6_driver_rds <- R6::R6Class( compress <- as.character(compress) } self$compress <- driver_rds_config(path, "compress", compress, - "TRUE", FALSE) + "TRUE", TRUE) self$compress <- parse_rds_compress(self$compress) if (!is.null(hash_algorithm)) { diff --git a/man/storr.Rd b/man/storr.Rd index 4d92b76..116857a 100644 --- a/man/storr.Rd +++ b/man/storr.Rd @@ -11,7 +11,7 @@ storr(driver, default_namespace = "objects") \item{default_namespace}{Default namespace to store objects in. By default \code{"objects"} is used, but this might be useful to -have two diffent \code{storr} objects pointing at the same +have two different \code{storr} objects pointing at the same underlying storage, but storing things in different namespaces.} } \description{ diff --git a/man/storr_rds.Rd b/man/storr_rds.Rd index a5c4e23..3dda5d3 100644 --- a/man/storr_rds.Rd +++ b/man/storr_rds.Rd @@ -18,7 +18,16 @@ for ephemeral storage, The \code{rappdirs} package (on CRAN) might be nice for persistent application data.} \item{compress}{Compress the generated file? This saves a small -amount of space for a reasonable amount of time.} +amount of space for a reasonable amount of time. Possible values +are \code{"none"} for no compression, \code{"gzfile"} to write +to a \code{gzfile} connection (default), and \code{"fst"} to use +\code{compress_fst()} (recommended, but requires the \code{fst} package). +To preserve back compatibility, \code{compress} can also be logical: +\code{TRUE} for \code{"gzfile"} and \code{FALSE} for \code{"none"}. +However, these values are not interchangeable for existings \code{storr}s. +For example, if you create a \code{storr} with \code{compress = TRUE}, +you must continue to use \code{compress = TRUE} +and not \code{compress = "gzfile"} when you recover it later.} \item{mangle_key}{Mangle keys? If TRUE, then the key is encoded using base64 before saving to the filesystem. See Details.} diff --git a/tests/testthat/test-driver-rds.R b/tests/testthat/test-driver-rds.R index bdd5f4d..9f4fe8e 100644 --- a/tests/testthat/test-driver-rds.R +++ b/tests/testthat/test-driver-rds.R @@ -138,12 +138,12 @@ test_that("large vector support", { file.remove(tmp) }) -test_that("compression support", { +test_that("gzfile compression support", { ## some data that will likely compress very well: data <- rep(1:10, each = 500) - st1 <- storr_rds(tempfile(), TRUE) - st2 <- storr_rds(tempfile(), FALSE) + st1 <- storr_rds(tempfile(), "gzfile") + st2 <- storr_rds(tempfile(), "none") on.exit({ st1$destroy() st2$destroy() @@ -182,10 +182,8 @@ test_that("backward compatibility: compression", { st <- storr_rds(path) expect_equal(st$list(), "key") expect_equal(st$get("key"), "value") - for (compress in list("gzfile", TRUE)) { - expect_silent(st <- storr_rds(path, compress = compress)) - } - for (compress in list("none", "fst", FALSE)) { + expect_silent(st <- storr_rds(path, compress = TRUE)) + for (compress in list("none", "gzfile", "fst", FALSE)) { expect_error(st <- storr_rds(path, compress = compress), "Incompatible value for compress") } @@ -194,10 +192,8 @@ test_that("backward compatibility: compression", { st <- storr_rds(path) expect_equal(st$list(), "key") expect_equal(st$get("key"), "value") - for (compress in list("none", FALSE)) { - expect_silent(st <- storr_rds(path, compress = compress)) - } - for (compress in list("gzfile", "fst", TRUE)) { + expect_silent(st <- storr_rds(path, compress = FALSE)) + for (compress in list("none", "gzfile", "fst", TRUE)) { expect_error(st <- storr_rds(path, compress = compress), "Incompatible value for compress") } From 67306bf786e4426c9bffd74d612a945dbc8b938d Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 09:09:47 -0400 Subject: [PATCH 04/14] Add failing test for fst compression --- DESCRIPTION | 1 + R/driver_rds.R | 7 +++++++ tests/testthat/test-driver-rds.R | 13 +++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 306def7..bd87e3b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,6 +25,7 @@ Suggests: DBI (>= 0.6), RSQLite (>= 1.1-2), RPostgres, + fst, knitr, mockr, parallel, diff --git a/R/driver_rds.R b/R/driver_rds.R index 18259f8..62b2495 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -469,6 +469,13 @@ parse_rds_compress <- function(compress) { "gzfile" } else if (identical(compress, "FALSE")) { "none" + } else if (identical(compress, "fst")) { + if (!requireNamespace("fst", quietly = TRUE)) { + stop( + "storr_rds(compress = \"fst\") requires the fst package.", + call. = FALSE + ) + } } else { compress } diff --git a/tests/testthat/test-driver-rds.R b/tests/testthat/test-driver-rds.R index 9f4fe8e..152421b 100644 --- a/tests/testthat/test-driver-rds.R +++ b/tests/testthat/test-driver-rds.R @@ -138,26 +138,35 @@ test_that("large vector support", { file.remove(tmp) }) -test_that("gzfile compression support", { +test_that("compression support", { + skip_if_not_installed("fst") + ## some data that will likely compress very well: data <- rep(1:10, each = 500) st1 <- storr_rds(tempfile(), "gzfile") st2 <- storr_rds(tempfile(), "none") + st3 <- storr_rds(tempfile(), "fst") + on.exit({ st1$destroy() st2$destroy() + st3$destroy() }) h1 <- st1$set("data", data) h2 <- st2$set("data", data) + h3 <- st3$set("data", data) - expect_identical(h1, h2) + expect_identical(h1, h2, h3) expect_gt(file.size(st2$driver$name_hash(h2)), file.size(st1$driver$name_hash(h1))) + expect_gt(file.size(st2$driver$name_hash(h2)), + file.size(st3$driver$name_hash(h3))) expect_identical(st1$get("data"), data) expect_identical(st2$get("data"), data) + expect_identical(st3$get("data"), data) }) test_that("backward compatibility", { From 482244bb21f31b74c5a6815f1879f7da1e4ac253 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 11:15:52 -0400 Subject: [PATCH 05/14] Fix #110 --- R/driver_rds.R | 13 ++++++------- R/hash.R | 16 +++++++++++++--- R/utils.R | 8 ++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index 62b2495..bcc925d 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -189,6 +189,7 @@ R6_driver_rds <- R6::R6Class( if (!is.null(compress)) { compress <- as.character(compress) } + self$compress <- driver_rds_config(path, "compress", compress, "TRUE", TRUE) self$compress <- parse_rds_compress(self$compress) @@ -222,7 +223,9 @@ R6_driver_rds <- R6::R6Class( }, get_object = function(hash) { - read_rds(self$name_hash(hash)) + out <- read_rds(self$name_hash(hash)) + if (identical(self$compress, "fst")) out <- fst::decompress_fst(out) + out }, set_object = function(hash, value) { @@ -470,12 +473,8 @@ parse_rds_compress <- function(compress) { } else if (identical(compress, "FALSE")) { "none" } else if (identical(compress, "fst")) { - if (!requireNamespace("fst", quietly = TRUE)) { - stop( - "storr_rds(compress = \"fst\") requires the fst package.", - call. = FALSE - ) - } + assert_fst() + compress } else { compress } diff --git a/R/hash.R b/R/hash.R index 1af0865..d159ed4 100644 --- a/R/hash.R +++ b/R/hash.R @@ -86,8 +86,20 @@ write_serialized_rds <- function(value, filename, compress, try_write_serialized_rds <- function(value, filename, compress, scratch_dir = NULL, long = 2^31 - 2) { tmp <- tempfile(tmpdir = scratch_dir %||% tempdir()) + if (identical(compress, "fst")) { + write_tmp_fst(value, tmp) + } else { + write_tmp_default(value, tmp, compress, scratch_dir, long) + } + file.rename(tmp, filename) +} - con <- (if (identical(compress, "gzfile")) gzfile else file)(tmp, "wb") +write_tmp_fst <- function(value, filename) { + saveRDS(fst::compress_fst(value), filename, compress = FALSE) +} + +write_tmp_default <- function(value, filename, compress, scratch_dir, long) { + con <- (if (identical(compress, "gzfile")) gzfile else file)(filename, "wb") needs_close <- TRUE on.exit(if (needs_close) close(con), add = TRUE) len <- length(value) @@ -99,10 +111,8 @@ try_write_serialized_rds <- function(value, filename, compress, } close(con) needs_close <- FALSE - file.rename(tmp, filename) } - ## Same pattern for write_lines. The difference is that this will ## delete the key on a failed write (otherwise there's a copy ## involved) diff --git a/R/utils.R b/R/utils.R index 256f157..b19bfbb 100644 --- a/R/utils.R +++ b/R/utils.R @@ -116,6 +116,14 @@ assert_probably_storr_driver <- function(x, name = deparse(substitute(x))) { invisible(x) } +assert_fst <- function() { + if (!requireNamespace("fst", quietly = TRUE)) { + stop( + "storr_rds(compress = \"fst\") requires the fst package.", + call. = FALSE + ) + } +} match_value <- function(x, choices, name = deparse(substitute(x))) { assert_scalar_character(x, name) From 2849c1e93b9e1701db6d13bc0c11301f4a288e36 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 11:26:08 -0400 Subject: [PATCH 06/14] Update news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index b5b0ab6..966861f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ ## storr 1.2.2 (2018-??-??) * Speed up the `$get_hash()` method of RDS drivers using C code and traits (#96, #98, @wlandau). +* Implement `fst` compression for RDS drivers (#110, #111, @wlandau, @MarcusKlik). ## storr 1.2.1 (2018-10-18) From d4d55553e43dcb6ae8c0f7f02e8fdce67b9f52b0 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 11:30:06 -0400 Subject: [PATCH 07/14] Use gzfile as default compression --- R/driver_rds.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index bcc925d..d537eb9 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -166,7 +166,7 @@ R6_driver_rds <- R6::R6Class( if (!is_new && !file.exists(driver_rds_config_file(path, "version"))) { write_if_missing("1.2.2", driver_rds_config_file(path, "version")) write_if_missing("TRUE", driver_rds_config_file(path, "mangle_key_pad")) - write_if_missing("TRUE", driver_rds_config_file(path, "compress")) + write_if_missing("gzfile", driver_rds_config_file(path, "compress")) write_if_missing("md5", driver_rds_config_file(path, "hash_algorithm")) } ## Then write out the version number: From c66c2c4aca1c6394f95c3a1e2e9ad46332653a2a Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Tue, 18 Jun 2019 11:55:49 -0400 Subject: [PATCH 08/14] Unserialize after decompress_fst() --- R/driver_rds.R | 5 ++++- tests/testthat/test-driver-rds.R | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index d537eb9..7f3956c 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -224,7 +224,10 @@ R6_driver_rds <- R6::R6Class( get_object = function(hash) { out <- read_rds(self$name_hash(hash)) - if (identical(self$compress, "fst")) out <- fst::decompress_fst(out) + if (identical(self$compress, "fst")) { + out <- fst::decompress_fst(out) + out <- unserialize(out) + } out }, diff --git a/tests/testthat/test-driver-rds.R b/tests/testthat/test-driver-rds.R index 152421b..d951a3b 100644 --- a/tests/testthat/test-driver-rds.R +++ b/tests/testthat/test-driver-rds.R @@ -154,9 +154,9 @@ test_that("compression support", { st3$destroy() }) - h1 <- st1$set("data", data) - h2 <- st2$set("data", data) - h3 <- st3$set("data", data) + h1 <- st1$set("data", data, use_cache = FALSE) + h2 <- st2$set("data", data, use_cache = FALSE) + h3 <- st3$set("data", data, use_cache = FALSE) expect_identical(h1, h2, h3) expect_gt(file.size(st2$driver$name_hash(h2)), @@ -164,9 +164,9 @@ test_that("compression support", { expect_gt(file.size(st2$driver$name_hash(h2)), file.size(st3$driver$name_hash(h3))) - expect_identical(st1$get("data"), data) - expect_identical(st2$get("data"), data) - expect_identical(st3$get("data"), data) + expect_identical(st1$get("data", use_cache = FALSE), data) + expect_identical(st2$get("data", use_cache = FALSE), data) + expect_identical(st3$get("data", use_cache = FALSE), data) }) test_that("backward compatibility", { From 486b1eadb70ff77e00512f834fe764b1372ae8bf Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Thu, 20 Jun 2019 08:56:41 -0400 Subject: [PATCH 09/14] Expose compression algo and compression factor --- R/driver_rds.R | 50 +++++++++++++++++++++++--------- R/hash.R | 17 ++++++----- man/storr_rds.Rd | 47 +++++++++++++++++++++--------- tests/testthat/test-driver-rds.R | 18 ++++++++---- 4 files changed, 91 insertions(+), 41 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index 7f3956c..de7f7d2 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -55,17 +55,36 @@ ##' might be nice for persistent application data. ##' ##' @param compress Compress the generated file? This saves a small -##' amount of space for a reasonable amount of time. Possible values +##' amount of space for a reasonable amount of time. Possible values: +##' \describe{ +##' \item{"none"}{No compression.} +##' \item{"gzip"}{gzip compression with \code{base::gzfile()} (default).} +##' \item{"lz4"}{ +##' lz4 compression via \code{fst::compress_fst()}. +##' Very fast, but lower quality than zstd. +##' } +##' \item{"zstd"}{ +##' zstd compression via \code{fst::compress_fst()}. +##' Higher quality but slower than lz4. +##' } +##' } +##' Choices \code{"lz4"} and \code{"} ##' are \code{"none"} for no compression, \code{"gzfile"} to write -##' to a \code{gzfile} connection (default), and \code{"fst"} to use +##' to a \code{gzfile} connection (default), and \code{"lz4"} to use ##' \code{compress_fst()} (recommended, but requires the \code{fst} package). -##' To preserve back compatibility, \code{compress} can also be logical: +##' +##' To preserve compatibility with earlier \code{storr}s, +##' \code{compress} can also be logical: ##' \code{TRUE} for \code{"gzfile"} and \code{FALSE} for \code{"none"}. ##' However, these values are not interchangeable for existings \code{storr}s. ##' For example, if you create a \code{storr} with \code{compress = TRUE}, ##' you must continue to use \code{compress = TRUE} ##' and not \code{compress = "gzfile"} when you recover it later. ##' +##' @param compression Numeric compression factor for \code{fst::compress_fst()}. +##' Between 0 and 100: 0 for lowest compression, 100 for maximum compression. +##' Only applies to \code{compress = "lz4"} and \code{compress = "zstd"}. +##' ##' @param mangle_key Mangle keys? If TRUE, then the key is encoded ##' using base64 before saving to the filesystem. See Details. ##' @@ -118,18 +137,18 @@ ##' # Clean up the two storrs: ##' st$destroy() ##' st2$destroy() -storr_rds <- function(path, compress = NULL, mangle_key = NULL, +storr_rds <- function(path, compress = NULL, compression = NULL, mangle_key = NULL, mangle_key_pad = NULL, hash_algorithm = NULL, default_namespace = "objects") { - storr(driver_rds(path, compress, mangle_key, mangle_key_pad, hash_algorithm), + storr(driver_rds(path, compress, compression, mangle_key, mangle_key_pad, hash_algorithm), default_namespace) } ##' @export ##' @rdname storr_rds -driver_rds <- function(path, compress = NULL, mangle_key = NULL, +driver_rds <- function(path, compress = NULL, compression = NULL, mangle_key = NULL, mangle_key_pad = NULL, hash_algorithm = NULL) { - R6_driver_rds$new(path, compress, mangle_key, mangle_key_pad, hash_algorithm) + R6_driver_rds$new(path, compress, compression, mangle_key, mangle_key_pad, hash_algorithm) } R6_driver_rds <- R6::R6Class( @@ -140,13 +159,14 @@ R6_driver_rds <- R6::R6Class( path = NULL, path_scratch = NULL, compress = NULL, + compression = NULL, mangle_key = NULL, mangle_key_pad = NULL, hash_algorithm = NULL, hash_length = NULL, traits = list(accept = "raw", throw_missing = TRUE), - initialize = function(path, compress, mangle_key, mangle_key_pad, + initialize = function(path, compress, compression, mangle_key, mangle_key_pad, hash_algorithm) { is_new <- !file.exists(file.path(path, "config")) dir_create(path) @@ -166,7 +186,8 @@ R6_driver_rds <- R6::R6Class( if (!is_new && !file.exists(driver_rds_config_file(path, "version"))) { write_if_missing("1.2.2", driver_rds_config_file(path, "version")) write_if_missing("TRUE", driver_rds_config_file(path, "mangle_key_pad")) - write_if_missing("gzfile", driver_rds_config_file(path, "compress")) + write_if_missing("gzip", driver_rds_config_file(path, "compress")) + write_if_missing("0", driver_rds_config_file(path, "compression")) write_if_missing("md5", driver_rds_config_file(path, "hash_algorithm")) } ## Then write out the version number: @@ -194,6 +215,9 @@ R6_driver_rds <- R6::R6Class( "TRUE", TRUE) self$compress <- parse_rds_compress(self$compress) + self$compression <- driver_rds_config(path, "compression", compression, + 0, FALSE) + if (!is.null(hash_algorithm)) { assert_scalar_character(hash_algorithm) } @@ -224,7 +248,7 @@ R6_driver_rds <- R6::R6Class( get_object = function(hash) { out <- read_rds(self$name_hash(hash)) - if (identical(self$compress, "fst")) { + if (self$compress %in% c("lz4", "zstd")) { out <- fst::decompress_fst(out) out <- unserialize(out) } @@ -236,7 +260,7 @@ R6_driver_rds <- R6::R6Class( ## already and avoids seralising twice. assert_raw(value) write_serialized_rds(value, self$name_hash(hash), self$compress, - self$path_scratch) + self$compression, self$path_scratch) }, exists_hash = function(key, namespace) { @@ -472,10 +496,10 @@ See 'Corrupt keys' within ?storr_rds for how to proceed" -> fmt parse_rds_compress <- function(compress) { if (identical(compress, "TRUE")) { - "gzfile" + "gzip" } else if (identical(compress, "FALSE")) { "none" - } else if (identical(compress, "fst")) { + } else if (compress %in% c("lz4", "zstd")) { assert_fst() compress } else { diff --git a/R/hash.R b/R/hash.R index d159ed4..3435b31 100644 --- a/R/hash.R +++ b/R/hash.R @@ -72,10 +72,10 @@ serialize_to_raw <- function(x, ascii, xdr) { ## been successful, otherwise the container will claim existence for ## an object which cannot be retrieved later on, causing havoc ## upstream. -write_serialized_rds <- function(value, filename, compress, +write_serialized_rds <- function(value, filename, compress, compression, scratch_dir = NULL, long = 2^31 - 2) { withCallingHandlers( - try_write_serialized_rds(value, filename, compress, scratch_dir, long), + try_write_serialized_rds(value, filename, compress, compression, scratch_dir, long), error = function(e) unlink(filename)) } @@ -83,23 +83,24 @@ write_serialized_rds <- function(value, filename, compress, ## The split here helps keep the order really consistent; we will ## close the connection on exit from try_write_serialized_rds and ## delete the file *after* that. -try_write_serialized_rds <- function(value, filename, compress, +try_write_serialized_rds <- function(value, filename, compress, compression, scratch_dir = NULL, long = 2^31 - 2) { tmp <- tempfile(tmpdir = scratch_dir %||% tempdir()) - if (identical(compress, "fst")) { - write_tmp_fst(value, tmp) + if (compress %in% c("lz4", "zstd")) { + write_tmp_fst(value, tmp, compress, compression) } else { write_tmp_default(value, tmp, compress, scratch_dir, long) } file.rename(tmp, filename) } -write_tmp_fst <- function(value, filename) { - saveRDS(fst::compress_fst(value), filename, compress = FALSE) +write_tmp_fst <- function(value, filename, compress, compression) { + saveRDS(fst::compress_fst(value, toupper(compress), compression), + filename, compress = FALSE) } write_tmp_default <- function(value, filename, compress, scratch_dir, long) { - con <- (if (identical(compress, "gzfile")) gzfile else file)(filename, "wb") + con <- (if (identical(compress, "gzip")) gzfile else file)(filename, "wb") needs_close <- TRUE on.exit(if (needs_close) close(con), add = TRUE) len <- length(value) diff --git a/man/storr_rds.Rd b/man/storr_rds.Rd index 3dda5d3..4f3b3cf 100644 --- a/man/storr_rds.Rd +++ b/man/storr_rds.Rd @@ -5,12 +5,12 @@ \alias{driver_rds} \title{rds object cache driver} \usage{ -storr_rds(path, compress = NULL, mangle_key = NULL, - mangle_key_pad = NULL, hash_algorithm = NULL, +storr_rds(path, compress = NULL, compression = NULL, + mangle_key = NULL, mangle_key_pad = NULL, hash_algorithm = NULL, default_namespace = "objects") -driver_rds(path, compress = NULL, mangle_key = NULL, - mangle_key_pad = NULL, hash_algorithm = NULL) +driver_rds(path, compress = NULL, compression = NULL, + mangle_key = NULL, mangle_key_pad = NULL, hash_algorithm = NULL) } \arguments{ \item{path}{Path for the store. \code{tempdir()} is a good choice @@ -18,16 +18,35 @@ for ephemeral storage, The \code{rappdirs} package (on CRAN) might be nice for persistent application data.} \item{compress}{Compress the generated file? This saves a small -amount of space for a reasonable amount of time. Possible values -are \code{"none"} for no compression, \code{"gzfile"} to write -to a \code{gzfile} connection (default), and \code{"fst"} to use -\code{compress_fst()} (recommended, but requires the \code{fst} package). -To preserve back compatibility, \code{compress} can also be logical: -\code{TRUE} for \code{"gzfile"} and \code{FALSE} for \code{"none"}. -However, these values are not interchangeable for existings \code{storr}s. -For example, if you create a \code{storr} with \code{compress = TRUE}, -you must continue to use \code{compress = TRUE} -and not \code{compress = "gzfile"} when you recover it later.} + amount of space for a reasonable amount of time. Possible values: + \describe{ + \item{"none"}{No compression.} + \item{"gzip"}{gzip compression with \code{base::gzfile()} (default).} + \item{"lz4"}{ + lz4 compression via \code{fst::compress_fst()}. + Very fast, but lower quality than zstd. + } + \item{"zstd"}{ + zstd compression via \code{fst::compress_fst()}. + Higher quality but slower than lz4. + } + } + Choices \code{"lz4"} and \code{"} + are \code{"none"} for no compression, \code{"gzfile"} to write + to a \code{gzfile} connection (default), and \code{"lz4"} to use + \code{compress_fst()} (recommended, but requires the \code{fst} package). + + To preserve compatibility with earlier \code{storr}s, + \code{compress} can also be logical: + \code{TRUE} for \code{"gzfile"} and \code{FALSE} for \code{"none"}. + However, these values are not interchangeable for existings \code{storr}s. + For example, if you create a \code{storr} with \code{compress = TRUE}, + you must continue to use \code{compress = TRUE} + and not \code{compress = "gzfile"} when you recover it later.} + +\item{compression}{Numeric compression factor for \code{fst::compress_fst()}. +Between 0 and 100: 0 for lowest compression, 100 for maximum compression. +Only applies to \code{compress = "lz4"} and \code{compress = "zstd"}.} \item{mangle_key}{Mangle keys? If TRUE, then the key is encoded using base64 before saving to the filesystem. See Details.} diff --git a/tests/testthat/test-driver-rds.R b/tests/testthat/test-driver-rds.R index d951a3b..f2b0f78 100644 --- a/tests/testthat/test-driver-rds.R +++ b/tests/testthat/test-driver-rds.R @@ -144,29 +144,35 @@ test_that("compression support", { ## some data that will likely compress very well: data <- rep(1:10, each = 500) - st1 <- storr_rds(tempfile(), "gzfile") + st1 <- storr_rds(tempfile(), "gzip") st2 <- storr_rds(tempfile(), "none") - st3 <- storr_rds(tempfile(), "fst") + st3 <- storr_rds(tempfile(), "lz4") + st4 <- storr_rds(tempfile(), "zstd") on.exit({ st1$destroy() st2$destroy() st3$destroy() + st4$destroy() }) h1 <- st1$set("data", data, use_cache = FALSE) h2 <- st2$set("data", data, use_cache = FALSE) h3 <- st3$set("data", data, use_cache = FALSE) + h4 <- st4$set("data", data, use_cache = FALSE) - expect_identical(h1, h2, h3) + expect_identical(h1, h2, h3, h4) expect_gt(file.size(st2$driver$name_hash(h2)), file.size(st1$driver$name_hash(h1))) expect_gt(file.size(st2$driver$name_hash(h2)), file.size(st3$driver$name_hash(h3))) - + expect_gt(file.size(st2$driver$name_hash(h2)), + file.size(st3$driver$name_hash(h4))) + expect_identical(st1$get("data", use_cache = FALSE), data) expect_identical(st2$get("data", use_cache = FALSE), data) expect_identical(st3$get("data", use_cache = FALSE), data) + expect_identical(st4$get("data", use_cache = FALSE), data) }) test_that("backward compatibility", { @@ -192,7 +198,7 @@ test_that("backward compatibility: compression", { expect_equal(st$list(), "key") expect_equal(st$get("key"), "value") expect_silent(st <- storr_rds(path, compress = TRUE)) - for (compress in list("none", "gzfile", "fst", FALSE)) { + for (compress in list("none", "gzip", "lz4", "zstd", FALSE)) { expect_error(st <- storr_rds(path, compress = compress), "Incompatible value for compress") } @@ -202,7 +208,7 @@ test_that("backward compatibility: compression", { expect_equal(st$list(), "key") expect_equal(st$get("key"), "value") expect_silent(st <- storr_rds(path, compress = FALSE)) - for (compress in list("none", "gzfile", "fst", TRUE)) { + for (compress in list("none", "gzip", "lz4", "zstd", TRUE)) { expect_error(st <- storr_rds(path, compress = compress), "Incompatible value for compress") } From 41cce5c009ee3a1be80b4f2e5a906270d8dfb577 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Thu, 20 Jun 2019 11:22:46 -0400 Subject: [PATCH 10/14] Use loadNamespace("fst") to check installation --- R/driver_rds.R | 2 +- R/utils.R | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index de7f7d2..57c9f93 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -500,7 +500,7 @@ parse_rds_compress <- function(compress) { } else if (identical(compress, "FALSE")) { "none" } else if (compress %in% c("lz4", "zstd")) { - assert_fst() + loadNamespace("fst") compress } else { compress diff --git a/R/utils.R b/R/utils.R index b19bfbb..a9afaeb 100644 --- a/R/utils.R +++ b/R/utils.R @@ -116,15 +116,6 @@ assert_probably_storr_driver <- function(x, name = deparse(substitute(x))) { invisible(x) } -assert_fst <- function() { - if (!requireNamespace("fst", quietly = TRUE)) { - stop( - "storr_rds(compress = \"fst\") requires the fst package.", - call. = FALSE - ) - } -} - match_value <- function(x, choices, name = deparse(substitute(x))) { assert_scalar_character(x, name) i <- match(x, choices) From bc7f36fb71773fe2bfe80a6f8cbba1eb6e43cb02 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Thu, 20 Jun 2019 11:25:44 -0400 Subject: [PATCH 11/14] Update news (compression) --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 966861f..694a49d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ ## storr 1.2.2 (2018-??-??) * Speed up the `$get_hash()` method of RDS drivers using C code and traits (#96, #98, @wlandau). -* Implement `fst` compression for RDS drivers (#110, #111, @wlandau, @MarcusKlik). +* Add support for `LZ4` and `ZSTD` compression (via `fst`) for RDS drivers (#110, #111, @wlandau, @MarcusKlik). ## storr 1.2.1 (2018-10-18) From df310a58d3d612fb4ddb3843b9bfb8ecfd58e969 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Thu, 20 Jun 2019 11:28:59 -0400 Subject: [PATCH 12/14] Remove some newlines in docstring --- R/driver_rds.R | 10 ++-------- man/storr_rds.Rd | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index 57c9f93..c72cee7 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -59,14 +59,8 @@ ##' \describe{ ##' \item{"none"}{No compression.} ##' \item{"gzip"}{gzip compression with \code{base::gzfile()} (default).} -##' \item{"lz4"}{ -##' lz4 compression via \code{fst::compress_fst()}. -##' Very fast, but lower quality than zstd. -##' } -##' \item{"zstd"}{ -##' zstd compression via \code{fst::compress_fst()}. -##' Higher quality but slower than lz4. -##' } +##' \item{"lz4"}{lz4 compression via \code{fst::compress_fst()}. Very fast, but lower quality than zstd.} +##' \item{"zstd"}{zstd compression via \code{fst::compress_fst()}. Higher quality but slower than lz4.} ##' } ##' Choices \code{"lz4"} and \code{"} ##' are \code{"none"} for no compression, \code{"gzfile"} to write diff --git a/man/storr_rds.Rd b/man/storr_rds.Rd index 4f3b3cf..7e1a54b 100644 --- a/man/storr_rds.Rd +++ b/man/storr_rds.Rd @@ -22,14 +22,8 @@ might be nice for persistent application data.} \describe{ \item{"none"}{No compression.} \item{"gzip"}{gzip compression with \code{base::gzfile()} (default).} - \item{"lz4"}{ - lz4 compression via \code{fst::compress_fst()}. - Very fast, but lower quality than zstd. - } - \item{"zstd"}{ - zstd compression via \code{fst::compress_fst()}. - Higher quality but slower than lz4. - } + \item{"lz4"}{lz4 compression via \code{fst::compress_fst()}. Very fast, but lower quality than zstd.} + \item{"zstd"}{zstd compression via \code{fst::compress_fst()}. Higher quality but slower than lz4.} } Choices \code{"lz4"} and \code{"} are \code{"none"} for no compression, \code{"gzfile"} to write From d0c640e2419cd3fa7a04b28e205ecac573a20303 Mon Sep 17 00:00:00 2001 From: wlandau Date: Sun, 23 Jun 2019 16:59:29 -0400 Subject: [PATCH 13/14] Use a couple custom error conditions --- tests/testthat/test-driver-rds.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-driver-rds.R b/tests/testthat/test-driver-rds.R index f2b0f78..4a54bf0 100644 --- a/tests/testthat/test-driver-rds.R +++ b/tests/testthat/test-driver-rds.R @@ -200,7 +200,8 @@ test_that("backward compatibility: compression", { expect_silent(st <- storr_rds(path, compress = TRUE)) for (compress in list("none", "gzip", "lz4", "zstd", FALSE)) { expect_error(st <- storr_rds(path, compress = compress), - "Incompatible value for compress") + "Incompatible value for compress", + class = "ConfigError") } path <- copy_to_tmp("v1.2.1_compress_FALSE") @@ -210,7 +211,8 @@ test_that("backward compatibility: compression", { expect_silent(st <- storr_rds(path, compress = FALSE)) for (compress in list("none", "gzip", "lz4", "zstd", TRUE)) { expect_error(st <- storr_rds(path, compress = compress), - "Incompatible value for compress") + "Incompatible value for compress", + class = "ConfigError") } }) From 245a9bd988e9d560822c147355d1ef1aaf80f3a6 Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Wed, 10 Jul 2019 10:27:07 -0400 Subject: [PATCH 14/14] Docs: lz4 and zstd use more memory --- R/driver_rds.R | 7 +++---- man/storr_rds.Rd | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/R/driver_rds.R b/R/driver_rds.R index c72cee7..e176598 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -62,10 +62,9 @@ ##' \item{"lz4"}{lz4 compression via \code{fst::compress_fst()}. Very fast, but lower quality than zstd.} ##' \item{"zstd"}{zstd compression via \code{fst::compress_fst()}. Higher quality but slower than lz4.} ##' } -##' Choices \code{"lz4"} and \code{"} -##' are \code{"none"} for no compression, \code{"gzfile"} to write -##' to a \code{gzfile} connection (default), and \code{"lz4"} to use -##' \code{compress_fst()} (recommended, but requires the \code{fst} package). +##' \code{"lz4"} is generally much faster than the other compression methods. +##' \code{"lz4"} and \code{"zstd"} require the \code{fst} package, +##' and they both use more memory (RAM) than the other choices. ##' ##' To preserve compatibility with earlier \code{storr}s, ##' \code{compress} can also be logical: diff --git a/man/storr_rds.Rd b/man/storr_rds.Rd index 7e1a54b..a5751a6 100644 --- a/man/storr_rds.Rd +++ b/man/storr_rds.Rd @@ -25,10 +25,9 @@ might be nice for persistent application data.} \item{"lz4"}{lz4 compression via \code{fst::compress_fst()}. Very fast, but lower quality than zstd.} \item{"zstd"}{zstd compression via \code{fst::compress_fst()}. Higher quality but slower than lz4.} } - Choices \code{"lz4"} and \code{"} - are \code{"none"} for no compression, \code{"gzfile"} to write - to a \code{gzfile} connection (default), and \code{"lz4"} to use - \code{compress_fst()} (recommended, but requires the \code{fst} package). + \code{"lz4"} is generally much faster than the other compression methods. + \code{"lz4"} and \code{"zstd"} require the \code{fst} package, + and they both use more memory (RAM) than the other choices. To preserve compatibility with earlier \code{storr}s, \code{compress} can also be logical: