diff --git a/.Rbuildignore b/.Rbuildignore index 6bb4f6f..e7f4089 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -20,3 +20,5 @@ Rprof\.out ^\.Rproj\.user$ ^doc$ ^Meta$ +^\.github$ +^_pkgdown\.yml$ diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 0000000..2d19fc7 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1 @@ +*.html diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml new file mode 100644 index 0000000..562fe0f --- /dev/null +++ b/.github/workflows/R-CMD-check.yaml @@ -0,0 +1,51 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + +name: R-CMD-check.yaml + +permissions: read-all + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: macos-latest, r: 'release'} + - {os: windows-latest, r: 'release'} + - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} + - {os: ubuntu-latest, r: 'release'} + - {os: ubuntu-latest, r: 'oldrel-1'} + + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_KEEP_PKG_SOURCE: yes + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + http-user-agent: ${{ matrix.config.http-user-agent }} + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - uses: r-lib/actions/check-r-package@v2 + with: + upload-snapshots: true + build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml new file mode 100644 index 0000000..bfc9f4d --- /dev/null +++ b/.github/workflows/pkgdown.yaml @@ -0,0 +1,49 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + release: + types: [published] + workflow_dispatch: + +name: pkgdown.yaml + +permissions: read-all + +jobs: + pkgdown: + runs-on: ubuntu-latest + # Only restrict concurrency for non-PR jobs + concurrency: + group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgdown, local::. + needs: website + + - name: Build site + run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) + shell: Rscript {0} + + - name: Deploy to GitHub pages 🚀 + if: github.event_name != 'pull_request' + uses: JamesIves/github-pages-deploy-action@v4.5.0 + with: + clean: false + branch: gh-pages + folder: docs diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml new file mode 100644 index 0000000..0ab748d --- /dev/null +++ b/.github/workflows/test-coverage.yaml @@ -0,0 +1,62 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + +name: test-coverage.yaml + +permissions: read-all + +jobs: + test-coverage: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::covr, any::xml2 + needs: coverage + + - name: Test coverage + run: | + cov <- covr::package_coverage( + quiet = FALSE, + clean = FALSE, + install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") + ) + print(cov) + covr::to_cobertura(cov) + shell: Rscript {0} + + - uses: codecov/codecov-action@v5 + with: + # Fail if error if not on PR, or if on PR and token is given + fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }} + files: ./cobertura.xml + plugins: noop + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Show testthat output + if: always() + run: | + ## -------------------------------------------------------------------- + find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload test results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: coverage-test-failures + path: ${{ runner.temp }}/package diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 38f5797..0000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: r -cache: packages -sudo: false - -r_packages: - - covr - -addons: - postgresql: "10" - apt: - packages: - - libpq-dev - -services: - - postgresql - -after_success: - - Rscript -e 'covr::codecov()' diff --git a/DESCRIPTION b/DESCRIPTION index 91fd6ff..6b270ea 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: storr Title: Simple Key Value Stores -Version: 1.2.5 +Version: 1.2.6 Description: Creates and manages simple key-value stores. These can use a variety of approaches for storing the data. This package implements the base methods and support for file system, in-memory @@ -8,7 +8,6 @@ Description: Creates and manages simple key-value stores. These can Depends: R (>= 3.3.0) License: MIT + file LICENSE -LazyData: true Authors@R: c(person("Rich", "FitzJohn", role = c("aut", "cre"), email = "rich.fitzjohn@gmail.com"), person(c("William", "Michael"), "Landau", @@ -16,7 +15,7 @@ Authors@R: c(person("Rich", "FitzJohn", role = c("aut", "cre"), role = "ctb", comment = c(ORCID = "0000-0003-1878-3253") )) -URL: https://github.com/richfitz/storr +URL: https://richfitz.github.io/storr/, https://github.com/richfitz/storr BugReports: https://github.com/richfitz/storr/issues Imports: R6 (>= 2.1.0), @@ -33,5 +32,7 @@ Suggests: rmarkdown, testthat (>= 3.0.0) VignetteBuilder: knitr -RoxygenNote: 6.1.1 +RoxygenNote: 7.3.2 Encoding: UTF-8 +Roxygen: list(markdown = TRUE) +Language: en-GB diff --git a/Makefile b/Makefile index 0aad5de..dbf7170 100644 --- a/Makefile +++ b/Makefile @@ -28,10 +28,6 @@ check_all: build autodoc: ${RSCRIPT} autodoc.R process -vignettes/storr.Rmd: vignettes_src/storr.R - ${RSCRIPT} -e 'library(sowsear); sowsear("$<", output="$@")' -vignettes/external.Rmd: vignettes_src/external.R - ${RSCRIPT} -e 'library(sowsear); sowsear("$<", output="$@")' vignettes: vignettes/storr.Rmd vignettes/external.Rmd ${RSCRIPT} -e 'library(methods); devtools::build_vignettes()' diff --git a/NEWS.md b/NEWS.md index 097ff9b..f7f97fc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,9 +16,9 @@ ## storr 1.1.3 (2017-12-15) * Only use version 2 serialisation, avoiding breakage when version 3 is released in 3.5.0 (#62, reported by Tomas Kalibera) -* Quote sql more safely (#60, reported by @wlandau) +* Quote SQL more safely (#60, reported by @wlandau) * Support for duplicating (`$duplicate`) and filling (`$fill`) keys with identical values, without duplicating or reserialising data (#56, requested by @wlandau) -* Moved to a shiny new [pkgdown](https://pkgdown.r-lib.org/) website - https://richfitz.github.com/storr +* Moved to a shiny new [`pkgdown`](https://pkgdown.r-lib.org/) website - https://richfitz.github.com/storr ## storr 1.1.2 (2017-09-08) @@ -33,18 +33,18 @@ ## storr 1.1.0 (2017-05-05) -* Support for using relational databases via DBI (SQLite, Postgres) #23 - MySQL and Microsoft SQL Server support is not implemented yet +* Support for using relational databases via DBI (`SQLite`, Postgres) #23 - MySQL and Microsoft SQL Server support is not implemented yet * Support for selecting hash algorithm at the driver level, so rather than being limited to md5, more robust algorithms can be used. This is implemented for all storr drivers, and for the rds driver will safely work on storr databases from before this version * `mget`/`mset`/`mset_by_value` added for vectorised `get`/`set`/`set_by_value` operations that reduce the number of roundtrips on some drivers. `del` and `exists` become vectorised at the same time. This requires extensive change to drivers but is backward compatible for users. * Large object support for rds is slightly improved (#25) but still very slow * Storr now supports exporting multiple namespaces at once (possibly all namespaces) making it easier migrate large amounts of data between storrs. This does cause a small breaking change where `$import()` now returns a matrix with columns `name`, `namespace` indicating what was imported (previously this was just a vector) * Storr import/export more gracefully handles missing keys -* RDS storrs no longer pad base64 filenames with trailing with `=` which reduces the number of possily problematic characters. Existing storrs are unaffected (#43) +* RDS storrs no longer pad base64 filenames with trailing with `=` which reduces the number of possibly problematic characters. Existing storrs are unaffected (#43) * RDS storrs are not left in an inconsistent state if RDS file writing fails (#40, thanks @krlmlr) ## storr 1.0.1 (2016-05-10) -* Updated to work with recent testthat (>= 1.0.0) +* Updated to work with recent `testthat` (>= 1.0.0) * Limited support for writing out very large serialised objects (rds) ## storr 1.0.0 (2016-01-19) diff --git a/R/base64.R b/R/base64.R index ad10230..b005712 100644 --- a/R/base64.R +++ b/R/base64.R @@ -7,7 +7,7 @@ ##' @param char62 Character to use for the 62nd index ##' @param char63 Character to use for the 63rd index ##' @param pad Logical, indicating if strings should be padded with -##' \code{=} characters (as RFC 4648 requires) +##' `=` characters (as RFC 4648 requires) ##' @export ##' @examples ##' x <- encode64("hello") @@ -47,7 +47,7 @@ encode64 <- function(x, char62 = "-", char63 = "_", pad = TRUE) { ##' @param error Throw an error if the decoding fails. If -##' \code{FALSE} then \code{NA_character_} values are returned for +##' `FALSE` then `NA_character_` values are returned for ##' failures. ##' ##' @export diff --git a/R/defunct.R b/R/defunct.R index 9be56b6..cac5ad7 100644 --- a/R/defunct.R +++ b/R/defunct.R @@ -1,13 +1,13 @@ ##' Defunct functions ##' -##' The redis functions (\code{driver_redis_api} and -##' \code{storr_redis_api}) have been moved out of this package and -##' into Redis. I don't believe anyone is using them at the time of -##' the move so this is being done fairly abruptly - this is -##' unfortunate, but necessary to avoid a circular dependency! The -##' new functions are simply \code{redux::driver_redis_api} and -##' \code{redux::storr_redis_api}, along with a helper function -##' \code{redux::storr_hiredis} which also creates the connection. +##' The Redis functions (`driver_redis_api` and `storr_redis_api`) +##' have been moved out of this package and into `redux`. I don't +##' believe anyone is using them at the time of the move so this is +##' being done fairly abruptly - this is unfortunate, but necessary to +##' avoid a circular dependency! The new functions are simply +##' `redux::driver_redis_api` and `redux::storr_redis_api`, along with +##' a helper function `redux::storr_hiredis` which also creates the +##' connection. ##' ##' @title Defunct functions ##' @param ... parameters (now all dropped as dots) diff --git a/R/driver_dbi.R b/R/driver_dbi.R index cc2f437..90bab0a 100644 --- a/R/driver_dbi.R +++ b/R/driver_dbi.R @@ -1,9 +1,9 @@ ##' Object cache driver using the "DBI" package interface for storage. ##' This means that storr can work for any supported "DBI" driver -##' (though practically this works only for SQLite and Postgres until +##' (though practically this works only for `SQLite` and Postgres until ##' some MySQL dialect translation is done). To connect, you must -##' provide the \emph{driver} object (e.g., \code{RSQLite::SQLite()}, -##' or \code{RPostgres::Postgres()} as the first argument. +##' provide the *driver* object (e.g., `RSQLite::SQLite()`, +##' or `RPostgres::Postgres()` as the first argument. ##' ##' Because the DBI package specifies a uniform interface for the ##' using DBI compliant databases, you need only to provide a @@ -17,7 +17,7 @@ ##' writing). Apart from that the names do not matter. ##' ##' Because of treatment of binary data by the underlying DBI drivers, -##' binary serialistion is not any faster (and might be slightly +##' binary serialisation is not any faster (and might be slightly ##' slower than) string serialisation, in contrast with my experience ##' with other backends. ##' @@ -25,10 +25,10 @@ ##' keys, namespaces and values into the database - this should allow ##' odd characters without throwing SQL syntax errors. Table names ##' can't be interpolated in the same way - these storr simply quotes, -##' but validates beforehand to ensure that \code{tbl_data} and -##' \code{tbl_keys} do not contain quotes. +##' but validates beforehand to ensure that `tbl_data` and +##' `tbl_keys` do not contain quotes. ##' -##' Be aware that \code{$destroy()} will close the connection to the +##' Be aware that `$destroy()` will close the connection to the ##' database. ##' ##' @title DBI storr driver @@ -40,24 +40,23 @@ ##' @param con Either A DBI connection or a DBI driver (see example) ##' ##' @param args Arguments to pass, along with the driver, to -##' \code{DBI::dbConnect} if \code{con} is a driver. +##' `DBI::dbConnect` if `con` is a driver. ##' ##' @param binary Optional logical indicating if the values should be ##' stored in binary. If possible, this is both (potentially ##' faster) and more accurate. However, at present it is supported -##' only under very recent DBI and RSQLite packages, and for no +##' only under very recent `DBI` and `RSQLite` packages, and for no ##' other DBI drivers, and is not actually any faster. If not given -##' (i.e., \code{NULL}), then binary storage will be used where +##' (i.e., `NULL`), then binary storage will be used where ##' possible when creating new tables, and where tables exist, we ##' use whatever was used in the existing tables. ##' ##' @param hash_algorithm Name of the hash algorithm to use. Possible ##' values are "md5", "sha1", and others supported by -##' \code{\link{digest}}. If not given, then we will default to +##' [digest::digest]. If not given, then we will default to ##' "md5". ##' -##' @param default_namespace Default namespace (see -##' \code{\link{storr}}). +##' @param default_namespace Default namespace (see [storr]). ##' ##' @export ##' @examples diff --git a/R/driver_environment.R b/R/driver_environment.R index 15d32a8..3fe5bf7 100644 --- a/R/driver_environment.R +++ b/R/driver_environment.R @@ -1,6 +1,6 @@ ##' Fast but transient environment driver. This driver saves objects ##' in a local R environment, without serialisation. This makes -##' lookup fast but it cannot be saved across sesssions. The +##' lookup fast but it cannot be saved across sessions. The ##' environment storr can be made persistent by saving it out as a ##' file storr though. ##' @@ -9,14 +9,14 @@ ##' @param envir The environment to point the storr at. The default ##' creates an new empty environment which is generally the right ##' choice. However, if you want multiple environment storrs -##' pointing at the same environment then pass the \code{envir} +##' pointing at the same environment then pass the `envir` ##' argument along. ##' -##' @param default_namespace Default namespace (see \code{\link{storr}}). +##' @param default_namespace Default namespace (see [storr]). ##' ##' @param hash_algorithm Name of the hash algorithm to use. Possible ##' values are "md5", "sha1", and others supported by -##' \code{\link{digest}}. If not given, then we will default to +##' [digest::digest]. If not given, then we will default to ##' "md5". ##' ##' @export diff --git a/R/driver_external.R b/R/driver_external.R index 7909747..6d51afe 100644 --- a/R/driver_external.R +++ b/R/driver_external.R @@ -2,21 +2,21 @@ ##' will try to fetch from an external data source if a resource can ##' not be found locally. This works by checking to see if a key is ##' present in the storr (and if so returning it). If it is not -##' found, then the function \code{fetch_hook} is run to fetch it. +##' found, then the function `fetch_hook` is run to fetch it. ##' -##' See the vignette \code{vignette("external")} for much more detail. +##' See the vignette `vignette("external")` for much more detail. ##' This function is likely most useful for things like caching ##' resources from websites, or computing long-running quantities on ##' demand. ##' @title Storr that looks for external resources -##' @param storage_driver Another \code{storr} driver to handle the +##' @param storage_driver Another `storr` driver to handle the ##' actual storage. ##' @param fetch_hook A function to run to fetch data when a key is ##' not found in the store. This function must take arguments -##' \code{key} and \code{namespace} and return an R object. It must +##' `key` and `namespace` and return an R object. It must ##' throw an error if the external resource cannot be resolved. ##' @param default_namespace Default namespace (see -##' \code{\link{storr}}) +##' [storr]) ##' @export storr_external <- function(storage_driver, fetch_hook, default_namespace = "objects") { @@ -73,18 +73,18 @@ check_external_fetch_hook <- function(fetch_hook) { ##' driver_external. We take two functions as arguments: the first ##' converts a key/namespace pair into a filename, and the second ##' reads from that filename. Because many R functions support -##' reading from URLs \code{fetch_hook_read} can be used to read from +##' reading from URLs `fetch_hook_read` can be used to read from ##' remote resources. ##' ##' For more information about using this, see -##' \code{\link{storr_external}} (this can be used as a -##' \code{fetch_hook} argument) and the vignette: -##' \code{vignette("external")} +##' [storr_external] (this can be used as a +##' `fetch_hook` argument) and the vignette: +##' `vignette("external")` ##' @title Hook to fetch a resource from a file. -##' @param fpath Function to convert \code{key, namespace} into a file +##' @param fpath Function to convert `key, namespace` into a file ##' path -##' @param fread Function for converting \code{filename} into an R -##' pobject +##' @param fread Function for converting `filename` into an R +##' object ##' @export ##' @examples ##' hook <- fetch_hook_read( diff --git a/R/driver_rds.R b/R/driver_rds.R index 3f1528b..ae0ab1b 100644 --- a/R/driver_rds.R +++ b/R/driver_rds.R @@ -1,8 +1,8 @@ ##' Object cache driver that saves objects using R's native -##' serialized file format (see \code{\link{saveRDS}}) on the +##' serialized file format (see [saveRDS]) on the ##' filesystem. ##' -##' The \code{mangle_key} argument will run each key that is created +##' The `mangle_key` argument will run each key that is created ##' through a "base 64" encoding. This means that keys that include ##' symbols that are invalid on filesystems (e.g, "/", ":") will be ##' replaced by harmless characters. The RFC 4648 dialect is used @@ -12,46 +12,46 @@ ##' with unmangled keys but the names of the stored files will be ##' different. ##' -##' Note that the \emph{namespace} is not mangled (at least not yet) +##' Note that the (*namespace* is not mangled (at least not yet) ##' so needs to contain characters that are valid in a filename. ##' ##' Because the actual file will be stored with mangled names it is ##' not safe to use the same path for a storr with and without ##' mangling. So once an rds storr has been created its "mangledness" -##' is set. Using \code{mangle_key = NULL} uses whatever mangledness +##' is set. Using `mangle_key = NULL` uses whatever mangledness ##' exists (or no mangledness if creating a new storr). ##' ##' @section Corrupt keys: ##' -##' Some file synchronisation utilities like dropbox can create file +##' Some file synchronisation utilities like Dropbox can create file ##' that confuse an rds storr (e.g., -##' \code{"myobject (Someone's conflicted copy)"}. If -##' \code{mangle_key} is \code{FALSE} these cannot be detected but at +##' `"myobject (Someone's conflicted copy)"`. If +##' `mangle_key` is `FALSE` these cannot be detected but at ##' the same time are not a real problem for storr. However, if -##' \code{mangle_key} is \code{TRUE} and keys are base64 encoded then +##' `mangle_key` is `TRUE` and keys are base64 encoded then ##' these conflicted copies can break parts of storr. ##' ##' If you see a warning asking you to deal with these files, please ##' delete the offending files; the path will be printed along with ##' the files that are causing the problem. ##' -##' Alternatively, you can try (assuming a storr object \code{st}) +##' Alternatively, you can try (assuming a storr object `st`) ##' running ##' -##' \preformatted{ +##' ``` ##' st$driver$purge_corrupt_keys() -##' } +##' ``` ##' ##' which will delete corrupted keys with no confirmation. The ##' messages that are printed to screen will be printed by default at ##' most once per minute per namespace. You can control this by -##' setting the R option \code{storr.corrupt.notice.period} - setting -##' this to \code{NA} suppresses the notice and otherwise it is +##' setting the R option `storr.corrupt.notice.period` - setting +##' this to `NA` suppresses the notice and otherwise it is ##' interpreted as the number of seconds. ##' ##' @title rds object cache driver -##' @param path Path for the store. \code{tempdir()} is a good choice -##' for ephemeral storage, The \code{rappdirs} package (on CRAN) +##' @param path Path for the store. `tempdir()` is a good choice +##' for ephemeral storage, The `rappdirs` package (on CRAN) ##' might be nice for persistent application data. ##' ##' @param compress Compress the generated file? This saves a small @@ -61,21 +61,19 @@ ##' using base64 before saving to the filesystem. See Details. ##' ##' @param mangle_key_pad Logical indicating if the filenames created -##' when using \code{mangle_key} should also be "padded" with the -##' \code{=} character to make up a round number of bytes. Padding -##' is required to satisfy the document that describes base64 -##' encoding (RFC 4648) but can cause problems in some applications -##' (see \href{https://github.com/richfitz/storr/issues/43}{this -##' issue}. The default is to not pad \emph{new} storr archives. -##' This should be generally safe to leave alone. +##' when using `mangle_key` should also be "padded" with the `=` +##' character to make up a round number of bytes. Padding is +##' required to satisfy the document that describes base64 encoding +##' (RFC 4648) but can cause problems in some applications (see +##' [this issue](https://github.com/richfitz/storr/issues/43)). The +##' default is to not pad *new* storr archives. This should be +##' generally safe to leave alone. ##' ##' @param hash_algorithm Name of the hash algorithm to use. Possible ##' values are "md5", "sha1", and others supported by -##' \code{\link{digest}}. If not given, then we will default to -##' "md5". +##' [digest::digest]. If not given, then we will default to "md5". ##' -##' @param default_namespace Default namespace (see -##' \code{\link{storr}}). +##' @param default_namespace Default namespace (see [storr]). ##' @export ##' @examples ##' diff --git a/R/driver_remote.R b/R/driver_remote.R index 403e0d2..66a347a 100644 --- a/R/driver_remote.R +++ b/R/driver_remote.R @@ -2,11 +2,11 @@ ##' location. This is the abstract interface (which does not do ##' anything useful) but which can be used with file operation driver ##' to store files elsewhere. This is not intended for end-user use -##' so there is no \code{storr_remote} function. Instead this +##' so there is no `storr_remote` function. Instead this ##' function is designed to support external packages that implement ##' the details. For a worked example, see the package tests -##' (\code{helper-remote.R}). In the current implementation these -##' build off of the \code{\link{driver_rds}} driver by copying files +##' (`helper-remote.R`). In the current implementation these +##' build off of the [driver_rds] driver by copying files ##' to some remote location. ##' ##' @title Remote storr @@ -14,14 +14,13 @@ ##' @param ops A file operations object. See tests for now to see ##' what is required to implement one. ##' -##' @param ... Arguments to pass through to \code{\link{driver_rds}}, -##' including \code{compress}, \code{mangle_key}, -##' \code{mangle_key_pad} and \code{hash_algorithm}. +##' @param ... Arguments to pass through to [driver_rds], including +##' `compress`, `mangle_key`, `mangle_key_pad` and `hash_algorithm`. ##' ##' @param path_local Path to a local cache. This can be left as -##' \code{NULL}, in which case a per-session cache will be used. +##' `NULL`, in which case a per-session cache will be used. ##' Alternatively, explicitly set to a path and the cache can be -##' reused over sessions. Only storr \emph{values} (i.e., objects) +##' reused over sessions. Only storr *values* (i.e., objects) ##' are cached - the key-to-value mapping is always fetched from the ##' remote storage. ##' diff --git a/R/multistorr.R b/R/multistorr.R index 892d07a..453e90e 100644 --- a/R/multistorr.R +++ b/R/multistorr.R @@ -17,8 +17,7 @@ ##' @title Storr with multiple storage drivers ##' @param keys Driver for the keys ##' @param data Driver for the data -##' @param default_namespace Default namespace (see -##' \code{\link{storr}}). +##' @param default_namespace Default namespace (see [storr]) ##' @export ##' @examples ##' # Create a storr that is stores keys in an environment and data in diff --git a/R/spec.R b/R/spec.R index ee24190..b4a571d 100644 --- a/R/spec.R +++ b/R/spec.R @@ -7,14 +7,13 @@ ##' to the test suite to guard against regressions. ##' ##' The test suite is included in the package as -##' \code{system.file("spec", package = "storr")}. +##' `system.file("spec", package = "storr")`. ##' ##' The procedure for each test block is: -##' \enumerate{ -##' \item{Create a new driver by running \code{dr <- create()}.} -##' \item{Run a number of tests.} -##' \item{Destroy the driver by running \code{dr$destroy()}.} -##' } +##' +##' 1. Create a new driver by running `dr <- create()` +##' 2. Run a number of tests. +##' 3. Destroy the driver by running `dr$destroy()` ##' ##' So before running this test suite, make sure this will not harm ##' any precious data! @@ -22,14 +21,14 @@ ##' @title Test a storr driver ##' ##' @param create A function with one arguments that when run with -##' \code{NULL} as the argument will create a new driver instance. -##' It will also be called with a driver (of the same type) as an +##' `NULL` as the argument will create a new driver instance. It +##' will also be called with a driver (of the same type) as an ##' argument - in this case, you must create a new driver object ##' pointing at the same underlying storage (see the examples). ##' Depending on your storage model, temporary directories, ##' in-memory locations, or random-but-unique prefixes may help ##' create isolated locations for the test (the tests assume that a -##' storr created with \code{create} is entirely empty). +##' storr created with `create` is entirely empty). ##' ##' @export ##' @examples diff --git a/R/storr.R b/R/storr.R index 96a5416..e817f8c 100644 --- a/R/storr.R +++ b/R/storr.R @@ -1,30 +1,30 @@ ##' Create an object cache; a "storr". A storr is a simple key-value -##' store where the actual content is stored in a content-addressible +##' store where the actual content is stored in a content-addressable ##' way (so that duplicate objects are only stored once) and with a ##' caching layer so that repeated lookups are fast even if the ##' underlying storage driver is slow. ##' ##' To create a storr you need to provide a "driver" object. There -##' are three in this package: \code{\link{driver_environment}} for -##' ephemeral in-memory storage, \code{\link{driver_rds}} for -##' serialized storage to disk, and \code{\link{driver_dbi}} for use -##' with DBI-compliant database interfaces. The \code{redux} package -##' (on CRAN) provides a storr driver that uses Redis. +##' are three in this package: [driver_environment] for ephemeral +##' in-memory storage, [driver_rds] for serialized storage to disk, +##' and [driver_dbi] for use with DBI-compliant database interfaces. +##' The `redux` package (on CRAN) provides a storr driver that uses +##' Redis. ##' -##' There are convenience functions (e.g., -##' \code{\link{storr_environment}} and \code{\link{storr_rds}}) that -##' may be more convenient to use than this function. +##' There are convenience functions (e.g., [storr_environment] and +##' [storr_rds]) that may be more convenient to use than this +##' function. ##' ##' Once a storr has been made it provides a number of methods. -##' Because storr uses \code{R6} (\code{\link{R6Class}}) objects, each -##' method is accessed by using \code{$} on a storr object (see the +##' Because storr uses `R6` ([R6::R6Class]) objects, each +##' method is accessed by using `$` on a storr object (see the ##' examples). The methods are described below in the "Methods" ##' section. ##' -##' The \code{default_namespace} affects all methods of the storr -##' object that refer to namespaces; if a namespace is not given, then -##' the action (get, set, del, list, import, export) will affect the -##' \code{default_namespace}. By default this is \code{"objects"}. +##' The `default_namespace` affects all methods of the storr object +##' that refer to namespaces; if a namespace is not given, then the +##' action (get, set, del, list, import, export) will affect the +##' `default_namespace`. By default this is `"objects"`. ##' ##' @template storr_methods ##' @@ -33,8 +33,8 @@ ##' @param driver A driver object ##' ##' @param default_namespace Default namespace to store objects in. -##' By default \code{"objects"} is used, but this might be useful to -##' have two different \code{storr} objects pointing at the same +##' By default `"objects"` is used, but this might be useful to have +##' two different `storr` objects pointing at the same ##' underlying storage, but storing things in different namespaces. ##' ##' @export @@ -60,7 +60,7 @@ ##' st$del("mykey") ##' ##' ## Storr objects can be created that have a default namespace that is -##' ## not "objects" by using the \code{default_namespace} argument (this +##' ## not "objects" by using the `default_namespace` argument (this ##' ## one also points at the same memory as the first storr). ##' st2 <- storr(driver_environment(st$driver$envir), ##' default_namespace = "namespace2") @@ -551,14 +551,14 @@ check_length <- function(key, namespace) { ##' Utility function for driver authors ##' ##' This exists to join, predictably, keys and namespaces for -##' operations like \code{mget}. Given a vector or scalar for -##' \code{key} and \code{namespace} we work out what the required -##' length is and recycle \code{key} and \code{namespace} to the +##' operations like `mget`. Given a vector or scalar for +##' `key` and `namespace` we work out what the required +##' length is and recycle `key` and `namespace` to the ##' appropriate length. ##' @title Recycle key and namespace ##' @param key A vector of keys ##' @param namespace A vector of namespace -##' @return A list with elements \code{n}, \code{key} and \code{namespace} +##' @return A list with elements `n`, `key` and `namespace` ##' @export join_key_namespace <- function(key, namespace) { n <- check_length(key, namespace) diff --git a/README.md b/README.md index 1b790f3..ba36751 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # storr + [![Project Status: Active - The project has reached a stable, usable state and is being actively developed.](http://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/) -[![Build Status](https://travis-ci.org/richfitz/storr.svg?branch=master)](https://travis-ci.org/richfitz/storr) -[![AppVeyor build status](https://ci.appveyor.com/api/projects/status/github/richfitz/storr?branch=master&svg=true)](https://ci.appveyor.com/project/richfitz/storr) -[![codecov.io](https://codecov.io/github/richfitz/storr/coverage.svg?branch=master)](https://codecov.io/github/richfitz/storr?branch=master) +[![R-CMD-check](https://github.com/richfitz/storr/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/richfitz/storr/actions/workflows/R-CMD-check.yaml) +[![Codecov test coverage](https://codecov.io/gh/richfitz/storr/graph/badge.svg)](https://app.codecov.io/gh/richfitz/storr) [![](http://www.r-pkg.org/badges/version/storr)](https://cran.r-project.org/package=storr) + Simple object cacher for R. `storr` acts as a very simple key-value store (supporting `get`/`set`/`del` for arbitrary R objects as data). The actual storage can be transient or persistent, local or distributed without changing the interface. To allow for distributed access, data is returned by *content* rather than simply by key (with a key/content lookup step) so that if another process changes the data, `storr` will retrieve the current version. @@ -12,14 +13,14 @@ Simple object cacher for R. `storr` acts as a very simple key-value store (supp * Content-addressable storage, storing and retrieving potentially fewer copies of identical data (useful if lookup is slow or over a network) and to make the system somewhat robust in the face of multiple accessing processes * Fetch from an external source (e.g. website) if a key is not found locally * Pluggable storage backends - currently - - environment (memory) - - rds (disk) - - [DBI](https://cran.r-project.org/package=DBI) though which you can use: - * [SQLite](https://sqlite.org/index.html) (via [RSQLite](https://cran.r-project.org/package=RSQLite)) - * [Postgres](https://www.postgresql.org/) (via [RPostgres](https://github.com/r-dbi/RPostgres)) + - `environment` (memory) + - `rds` (disk) + - [`DBI`](https://cran.r-project.org/package=DBI) though which you can use: + * [`SQLite`](https://sqlite.org/index.html) (via [`RSQLite`](https://cran.r-project.org/package=RSQLite)) + * [`Postgres`](https://www.postgresql.org/) (via [`RPostgres`](https://github.com/r-dbi/RPostgres)) - Redis (provided by [redux](https://github.com/richfitz/redux)) - - [rlite](https://github.com/seppo0010/rlite) (via [rrlite](https://github.com/ropensci/rrlite), interface function in [redux](https://github.com/richfitz/redux)) - - An on disk memory-mapped file interface is implemented in [thor](https://github.com/richfitz/thor) + - [`rlite`](https://github.com/seppo0010/rlite) (via [`rrlite`](https://github.com/ropensci/rrlite), interface function in [redux](https://github.com/richfitz/redux)) + - An on disk memory-mapped file interface is implemented in [`thor`](https://github.com/richfitz/thor) `storr` always goes back to the common storage (database, filesystem, whatever) for the current object to hash mapping, ensuring consistency when using multiple processes. However, when retrieving or writing the data given a hash we can often avoid accessing the underlying storage. This means that repeated lookups happen quickly while still being able to reflect changes elsewhere; time savings can be substantial where large objects are being stored. diff --git a/_pkgdown.yml b/_pkgdown.yml new file mode 100644 index 0000000..ad84a95 --- /dev/null +++ b/_pkgdown.yml @@ -0,0 +1,4 @@ +url: https://richfitz.github.io/storr + +template: + bootstrap: 5 diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index c6c1438..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,45 +0,0 @@ -# DO NOT CHANGE the "init" and "install" sections below - -# Download script file from GitHub -init: - ps: | - $ErrorActionPreference = "Stop" - Invoke-WebRequest http://raw.github.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" - Import-Module '..\appveyor-tool.ps1' - -install: - ps: Bootstrap - -cache: - - C:\RLibrary - -# Adapt as necessary starting from here - -build_script: - - travis-tool.sh install_deps - -test_script: - - travis-tool.sh run_tests - -on_failure: - - 7z a failure.zip *.Rcheck\* - - appveyor PushArtifact failure.zip - -artifacts: - - path: '*.Rcheck\**\*.log' - name: Logs - - - path: '*.Rcheck\**\*.out' - name: Logs - - - path: '*.Rcheck\**\*.fail' - name: Logs - - - path: '*.Rcheck\**\*.Rout' - name: Logs - - - path: '\*_*.tar.gz' - name: Bits - - - path: '\*_*.zip' - name: Bits diff --git a/inst/WORDLIST b/inst/WORDLIST new file mode 100644 index 0000000..acd1518 --- /dev/null +++ b/inst/WORDLIST @@ -0,0 +1,25 @@ +DBI's +Dropbox +Kalibera +Postgres +R's +Redis +backends +db +del +etc +mangledness +md +memoisation +memoise +memoised +nd +rd +rds +redux +reserialising +roundtrips +serialisation +sha +storrs +unmangled diff --git a/man-roxygen/storr.yml b/man-roxygen/storr.yml index 5f23310..cbd6b8e 100644 --- a/man-roxygen/storr.yml +++ b/man-roxygen/storr.yml @@ -24,7 +24,7 @@ set: Any R object to store. The object will generally be serialized (this is not actually true for the environment storr) so only objects that would usually be expected to survive a - \code{saveRDS}/\code{readRDS} roundtrip will work. This + `saveRDS`/`readRDS` roundtrip will work. This excludes Rcpp modules objects, external pointers, etc. But any "normal" R object will work fine. use_cache: &use_cache >- @@ -36,19 +36,19 @@ set: set_by_value: short: >- - Like \code{set} but saves the object with a key that is the same + Like `set` but saves the object with a key that is the same as the hash of the object. Equivalent to - \code{$set(digest::digest(value), value)}. + `$set(digest::digest(value), value)`. params: - value: An R object to save, with the same limitations as \code{set}. + value: An R object to save, with the same limitations as `set`. namespace: Optional namespace to save the key into. use_cache: *use_cache get: short: >- Retrieve an object from the storr. If the requested value is not - found then a \code{KeyError} will be raised (an R error, but can be - caught with \code{tryCatch}; see the "storr" vignette). + found then a `KeyError` will be raised (an R error, but can be + caught with `tryCatch`; see the "storr" vignette). params: key: &key_get The name of the key to get. namespace: &ns_get Optional namespace to look for the key within. @@ -61,14 +61,14 @@ get_hash: key: *key_get namespace: *ns_get del: - short: Delete an object fom the storr. + short: Delete an object from the storr. params: key: &key_vector A vector of names of keys namespace: &ns The namespace of the key. value: >- A logical vector the same length as the recycled length of - key/namespace, with each element being \code{TRUE} if an object - was deleted, \code{FALSE} otherwise. + key/namespace, with each element being `TRUE` if an object + was deleted, `FALSE` otherwise. duplicate: short: >- @@ -78,7 +78,7 @@ duplicate: copy of the data, just the pointer to the data (for more details see the storr vignette which explains the storage model in more detail). Multiple keys (and/or namespaces) can be provided, with - keys and nmespaces recycled as needed. However, the number of + keys and namespaces recycled as needed. However, the number of source and destination keys must be the same. The order of operation is not defined, so if the sets of keys are overlapping it is undefined behaviour. @@ -87,7 +87,7 @@ duplicate: key_dest: The destination key namespace: >- The namespace to copy keys within (used only of - \code{namespace_src} and \code{namespace_dest} are not provided + `namespace_src` and `namespace_dest` are not provided namespace_src: >- The source namespace - use this where keys are duplicated across namespaces. @@ -113,7 +113,7 @@ clear: implement a bulk clear method that will allow faster clearing. params: namespace: >- - A namespace, to clear a single namespace, or \code{NULL} to + A namespace, to clear a single namespace, or `NULL` to clear all namespaces. exists: short: Test if a key exists within a namespace @@ -122,8 +122,8 @@ exists: namespace: *ns value: >- A logical vector the same length as the recycled length of - key/namespace, with each element being \code{TRUE} if the object - exists and \code{FALSE} otherwise. + key/namespace, with each element being `TRUE` if the object + exists and `FALSE` otherwise. exists_object: short: Test if an object with a given hash exists within the storr params: @@ -137,7 +137,7 @@ mset: namespace: A vector of namespaces (either a single namespace or a vector) use_cache: *use_cache details: &recycling >- - The arguments \code{key} and \code{namespace} are recycled such + The arguments `key` and `namespace` are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. @@ -148,24 +148,24 @@ mget: namespace: A vector of namespaces (either a single namespace or a vector) use_cache: *use_cache missing: >- - Value to use for missing elements; by default \code{NULL} will - be used. IF \code{NULL} is a value that you might have stored + Value to use for missing elements; by default `NULL` will + be used. IF `NULL` is a value that you might have stored in the storr you might want to use a different value here to distinguish "missing" from "set to NULL". In addition, the - \code{missing} attribute will indicate which values were + `missing` attribute will indicate which values were missing. details: *recycling value: >- - A list with a length of the recycled length of \code{key} and - \code{namespace}. If any elements are missing, then an attribute - \code{missing} will indicate the elements that are missing (this + A list with a length of the recycled length of `key` and + `namespace`. If any elements are missing, then an attribute + `missing` will indicate the elements that are missing (this will be an integer vector with the indices of values were not found in the storr). mset_by_value: short: >- Set multiple elements at once, by value. A cross between - \code{mset} and \code{set_by_value}. + `mset` and `set_by_value`. params: value: A list or vector of values to set into the storr. namespace: A vector of namespaces @@ -176,7 +176,7 @@ gc: Garbage collect the storr. Because keys do not directly map to objects, but instead map to hashes which map to objects, it is possible that hash/object pairs can persist with nothing pointing - at them. Running \code{gc} will remove these objects from the + at them. Running `gc` will remove these objects from the storr. get_value: short: >- @@ -184,7 +184,7 @@ get_value: params: hash: The hash of the object to retrieve. use_cache: *use_cache - value: The object if it is present, otherwise throw a \code{HashError}. + value: The object if it is present, otherwise throw a `HashError`. set_value: short: >- Add an object value, but don't add a key. You will not need to @@ -220,18 +220,18 @@ import: Object to import objects from; can be a list, environment or another storr. list: >- - Names of of objects to import (or \code{NULL} to import all - objects in \code{envir}. If given it must be a character + Names of of objects to import (or `NULL` to import all + objects in `envir`. If given it must be a character vector. If named, the names of the character vector will be the names of the objects as created in the storr. namespace: >- Namespace to get objects from, and to put objects into. If - \code{NULL}, all namespaces from \code{src} will be imported. - If named, then the same rule is followed as \code{list}; - \code{namespace = c(a = b)} will import the contents of - namespace \code{b} as namespace \code{a}. + `NULL`, all namespaces from `src` will be imported. + If named, then the same rule is followed as `list`; + `namespace = c(a = b)` will import the contents of + namespace `b` as namespace `a`. skip_missing: >- - Logical, indicating if missing keys (specified in \code{list}) + Logical, indicating if missing keys (specified in `list`) should be skipped over, rather than being treated as an error (the default). @@ -240,54 +240,54 @@ export: params: dest: >- A target destination to export objects to; can be a list, - environment, or another storr. Use \code{list()} to export to a - brand new list, or use \code{as.list(object)} for a shorthand. + environment, or another storr. Use `list()` to export to a + brand new list, or use `as.list(object)` for a shorthand. list: >- - Names of objects to export, with the same rules as \code{list} - in \code{$import}. + Names of objects to export, with the same rules as `list` + in `$import`. namespace: >- Namespace to get objects from, and to put objects into. If - \code{NULL}, then this will export namespaces from this (source) + `NULL`, then this will export namespaces from this (source) storr into the destination; if there is more than one - namespace,this is only possible if \code{dest} is a storr + namespace,this is only possible if `dest` is a storr (otherwise there will be an error). skip_missing: >- - Logical, indicating if missing keys (specified in \code{list}) + Logical, indicating if missing keys (specified in `list`) should be skipped over, rather than being treated as an error (the default). value: >- - Invisibly, \code{dest}, which allows use of \code{e <- - st$export(new.env())} and \code{x <- st$export(list())}. + Invisibly, `dest`, which allows use of `e <- + st$export(new.env())` and `x <- st$export(list())`. archive_export: short: >- Export objects from the storr into a special "archive" storr, - which is an \code{\link{storr_rds}} with name mangling turned on - (which encodes keys with base64 so that they do not voilate + which is an [storr_rds] with name mangling turned on + (which encodes keys with base64 so that they do not violate filesystem naming conventions). params: path: Path to create the storr at; can exist already. - names: As for \code{$export} + names: As for `$export` namespace: >- - Namespace to get objects from. If \code{NULL}, then exports all + Namespace to get objects from. If `NULL`, then exports all namespaces found in this (source) storr. archive_import: short: >- - Inverse of \code{archive_export}; import objects from a storr that - was created by \code{archive_export}. + Inverse of `archive_export`; import objects from a storr that + was created by `archive_export`. params: path: Path of the exported storr. - names: As for \code{$import} + names: As for `$import` namespace: >- - Namespace to import objects into. If \code{NULL}, then imports + Namespace to import objects into. If `NULL`, then imports all namespaces from the source storr. index_export: short: >- Generate a data.frame with an index of objects present in a storr. This can be saved (for an rds storr) in lieu of the keys/ - directory and re-imported with \code{index_import}. It will + directory and re-imported with `index_import`. It will provide a more version control friendly export of the data in a storr. params: diff --git a/man-roxygen/storr_methods.R b/man-roxygen/storr_methods.R index 5c20293..71d644a 100644 --- a/man-roxygen/storr_methods.R +++ b/man-roxygen/storr_methods.R @@ -24,7 +24,7 @@ ##' \item{\code{key}: The key name. Can be any string. ##' } ##' -##' \item{\code{value}: Any R object to store. The object will generally be serialized (this is not actually true for the environment storr) so only objects that would usually be expected to survive a \code{saveRDS}/\code{readRDS} roundtrip will work. This excludes Rcpp modules objects, external pointers, etc. But any "normal" R object will work fine. +##' \item{\code{value}: Any R object to store. The object will generally be serialized (this is not actually true for the environment storr) so only objects that would usually be expected to survive a `saveRDS`/`readRDS` roundtrip will work. This excludes Rcpp modules objects, external pointers, etc. But any "normal" R object will work fine. ##' } ##' ##' \item{\code{namespace}: An optional namespace. By default the default namespace that the storr was created with will be used (by default that is "objects"). Different namespaces allow different types of objects to be stored without risk of names colliding. Use of namespaces is optional, but if used they must be a string. @@ -38,14 +38,14 @@ ##' Invisibly, the hash of the saved object. ##' } ##' \item{\code{set_by_value}}{ -##' Like \code{set} but saves the object with a key that is the same as the hash of the object. Equivalent to \code{$set(digest::digest(value), value)}. +##' Like `set` but saves the object with a key that is the same as the hash of the object. Equivalent to `$set(digest::digest(value), value)`. ##' ##' \emph{Usage:} ##' \code{set_by_value(value, namespace = self$default_namespace, use_cache = TRUE)} ##' ##' \emph{Arguments:} ##' \itemize{ -##' \item{\code{value}: An R object to save, with the same limitations as \code{set}. +##' \item{\code{value}: An R object to save, with the same limitations as `set`. ##' } ##' ##' \item{\code{namespace}: Optional namespace to save the key into. @@ -56,7 +56,7 @@ ##' } ##' } ##' \item{\code{get}}{ -##' Retrieve an object from the storr. If the requested value is not found then a \code{KeyError} will be raised (an R error, but can be caught with \code{tryCatch}; see the "storr" vignette). +##' Retrieve an object from the storr. If the requested value is not found then a `KeyError` will be raised (an R error, but can be caught with `tryCatch`; see the "storr" vignette). ##' ##' \emph{Usage:} ##' \code{get(key, namespace = self$default_namespace, use_cache = TRUE)} @@ -89,7 +89,7 @@ ##' } ##' } ##' \item{\code{del}}{ -##' Delete an object fom the storr. +##' Delete an object from the storr. ##' ##' \emph{Usage:} ##' \code{del(key, namespace = self$default_namespace)} @@ -104,10 +104,10 @@ ##' } ##' ##' \emph{Value}: -##' A logical vector the same length as the recycled length of key/namespace, with each element being \code{TRUE} if an object was deleted, \code{FALSE} otherwise. +##' A logical vector the same length as the recycled length of key/namespace, with each element being `TRUE` if an object was deleted, `FALSE` otherwise. ##' } ##' \item{\code{duplicate}}{ -##' Duplicate the value of a set of keys into a second set of keys. Because the value stored against a key is just the hash of its content, this operation is very efficient - it does not make a copy of the data, just the pointer to the data (for more details see the storr vignette which explains the storage model in more detail). Multiple keys (and/or namespaces) can be provided, with keys and nmespaces recycled as needed. However, the number of source and destination keys must be the same. The order of operation is not defined, so if the sets of keys are overlapping it is undefined behaviour. +##' Duplicate the value of a set of keys into a second set of keys. Because the value stored against a key is just the hash of its content, this operation is very efficient - it does not make a copy of the data, just the pointer to the data (for more details see the storr vignette which explains the storage model in more detail). Multiple keys (and/or namespaces) can be provided, with keys and namespaces recycled as needed. However, the number of source and destination keys must be the same. The order of operation is not defined, so if the sets of keys are overlapping it is undefined behaviour. ##' ##' \emph{Usage:} ##' \code{duplicate(key_src, key_dest, namespace = self$default_namespace, @@ -121,7 +121,7 @@ ##' \item{\code{key_dest}: The destination key ##' } ##' -##' \item{\code{namespace}: The namespace to copy keys within (used only of \code{namespace_src} and \code{namespace_dest} are not provided +##' \item{\code{namespace}: The namespace to copy keys within (used only of `namespace_src` and `namespace_dest` are not provided ##' } ##' ##' \item{\code{namespace_src}: The source namespace - use this where keys are duplicated across namespaces. @@ -160,7 +160,7 @@ ##' ##' \emph{Arguments:} ##' \itemize{ -##' \item{\code{namespace}: A namespace, to clear a single namespace, or \code{NULL} to clear all namespaces. +##' \item{\code{namespace}: A namespace, to clear a single namespace, or `NULL` to clear all namespaces. ##' } ##' } ##' } @@ -180,7 +180,7 @@ ##' } ##' ##' \emph{Value}: -##' A logical vector the same length as the recycled length of key/namespace, with each element being \code{TRUE} if the object exists and \code{FALSE} otherwise. +##' A logical vector the same length as the recycled length of key/namespace, with each element being `TRUE` if the object exists and `FALSE` otherwise. ##' } ##' \item{\code{exists_object}}{ ##' Test if an object with a given hash exists within the storr @@ -216,7 +216,7 @@ ##' } ##' ##' \emph{Details:} -##' The arguments \code{key} and \code{namespace} are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. +##' The arguments `key` and `namespace` are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. ##' } ##' \item{\code{mget}}{ ##' Get multiple elements at once @@ -236,18 +236,18 @@ ##' \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). ##' } ##' -##' \item{\code{missing}: Value to use for missing elements; by default \code{NULL} will be used. IF \code{NULL} is a value that you might have stored in the storr you might want to use a different value here to distinguish "missing" from "set to NULL". In addition, the \code{missing} attribute will indicate which values were missing. +##' \item{\code{missing}: Value to use for missing elements; by default `NULL` will be used. IF `NULL` is a value that you might have stored in the storr you might want to use a different value here to distinguish "missing" from "set to NULL". In addition, the `missing` attribute will indicate which values were missing. ##' } ##' } ##' ##' \emph{Details:} -##' The arguments \code{key} and \code{namespace} are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. +##' The arguments `key` and `namespace` are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. ##' ##' \emph{Value}: -##' A list with a length of the recycled length of \code{key} and \code{namespace}. If any elements are missing, then an attribute \code{missing} will indicate the elements that are missing (this will be an integer vector with the indices of values were not found in the storr). +##' A list with a length of the recycled length of `key` and `namespace`. If any elements are missing, then an attribute `missing` will indicate the elements that are missing (this will be an integer vector with the indices of values were not found in the storr). ##' } ##' \item{\code{mset_by_value}}{ -##' Set multiple elements at once, by value. A cross between \code{mset} and \code{set_by_value}. +##' Set multiple elements at once, by value. A cross between `mset` and `set_by_value`. ##' ##' \emph{Usage:} ##' \code{mset_by_value(value, namespace = self$default_namespace, use_cache = TRUE)} @@ -265,7 +265,7 @@ ##' } ##' } ##' \item{\code{gc}}{ -##' Garbage collect the storr. Because keys do not directly map to objects, but instead map to hashes which map to objects, it is possible that hash/object pairs can persist with nothing pointing at them. Running \code{gc} will remove these objects from the storr. +##' Garbage collect the storr. Because keys do not directly map to objects, but instead map to hashes which map to objects, it is possible that hash/object pairs can persist with nothing pointing at them. Running `gc` will remove these objects from the storr. ##' ##' \emph{Usage:} ##' \code{gc()} @@ -286,7 +286,7 @@ ##' } ##' ##' \emph{Value}: -##' The object if it is present, otherwise throw a \code{HashError}. +##' The object if it is present, otherwise throw a `HashError`. ##' } ##' \item{\code{set_value}}{ ##' Add an object value, but don't add a key. You will not need to use this very often, but it is used internally. @@ -366,13 +366,13 @@ ##' \item{\code{src}: Object to import objects from; can be a list, environment or another storr. ##' } ##' -##' \item{\code{list}: Names of of objects to import (or \code{NULL} to import all objects in \code{envir}. If given it must be a character vector. If named, the names of the character vector will be the names of the objects as created in the storr. +##' \item{\code{list}: Names of of objects to import (or `NULL` to import all objects in `envir`. If given it must be a character vector. If named, the names of the character vector will be the names of the objects as created in the storr. ##' } ##' -##' \item{\code{namespace}: Namespace to get objects from, and to put objects into. If \code{NULL}, all namespaces from \code{src} will be imported. If named, then the same rule is followed as \code{list}; \code{namespace = c(a = b)} will import the contents of namespace \code{b} as namespace \code{a}. +##' \item{\code{namespace}: Namespace to get objects from, and to put objects into. If `NULL`, all namespaces from `src` will be imported. If named, then the same rule is followed as `list`; `namespace = c(a = b)` will import the contents of namespace `b` as namespace `a`. ##' } ##' -##' \item{\code{skip_missing}: Logical, indicating if missing keys (specified in \code{list}) should be skipped over, rather than being treated as an error (the default). +##' \item{\code{skip_missing}: Logical, indicating if missing keys (specified in `list`) should be skipped over, rather than being treated as an error (the default). ##' } ##' } ##' } @@ -385,24 +385,24 @@ ##' ##' \emph{Arguments:} ##' \itemize{ -##' \item{\code{dest}: A target destination to export objects to; can be a list, environment, or another storr. Use \code{list()} to export to a brand new list, or use \code{as.list(object)} for a shorthand. +##' \item{\code{dest}: A target destination to export objects to; can be a list, environment, or another storr. Use `list()` to export to a brand new list, or use `as.list(object)` for a shorthand. ##' } ##' -##' \item{\code{list}: Names of objects to export, with the same rules as \code{list} in \code{$import}. +##' \item{\code{list}: Names of objects to export, with the same rules as `list` in `$import`. ##' } ##' -##' \item{\code{namespace}: Namespace to get objects from, and to put objects into. If \code{NULL}, then this will export namespaces from this (source) storr into the destination; if there is more than one namespace,this is only possible if \code{dest} is a storr (otherwise there will be an error). +##' \item{\code{namespace}: Namespace to get objects from, and to put objects into. If `NULL`, then this will export namespaces from this (source) storr into the destination; if there is more than one namespace,this is only possible if `dest` is a storr (otherwise there will be an error). ##' } ##' -##' \item{\code{skip_missing}: Logical, indicating if missing keys (specified in \code{list}) should be skipped over, rather than being treated as an error (the default). +##' \item{\code{skip_missing}: Logical, indicating if missing keys (specified in `list`) should be skipped over, rather than being treated as an error (the default). ##' } ##' } ##' ##' \emph{Value}: -##' Invisibly, \code{dest}, which allows use of \code{e <- st$export(new.env())} and \code{x <- st$export(list())}. +##' Invisibly, `dest`, which allows use of `e <- st$export(new.env())` and `x <- st$export(list())`. ##' } ##' \item{\code{archive_export}}{ -##' Export objects from the storr into a special "archive" storr, which is an \code{\link{storr_rds}} with name mangling turned on (which encodes keys with base64 so that they do not voilate filesystem naming conventions). +##' Export objects from the storr into a special "archive" storr, which is an [storr_rds] with name mangling turned on (which encodes keys with base64 so that they do not violate filesystem naming conventions). ##' ##' \emph{Usage:} ##' \code{archive_export(path, names = NULL, namespace = NULL)} @@ -412,15 +412,15 @@ ##' \item{\code{path}: Path to create the storr at; can exist already. ##' } ##' -##' \item{\code{names}: As for \code{$export} +##' \item{\code{names}: As for `$export` ##' } ##' -##' \item{\code{namespace}: Namespace to get objects from. If \code{NULL}, then exports all namespaces found in this (source) storr. +##' \item{\code{namespace}: Namespace to get objects from. If `NULL`, then exports all namespaces found in this (source) storr. ##' } ##' } ##' } ##' \item{\code{archive_import}}{ -##' Inverse of \code{archive_export}; import objects from a storr that was created by \code{archive_export}. +##' Inverse of `archive_export`; import objects from a storr that was created by `archive_export`. ##' ##' \emph{Usage:} ##' \code{archive_import(path, names = NULL, namespace = NULL)} @@ -430,15 +430,15 @@ ##' \item{\code{path}: Path of the exported storr. ##' } ##' -##' \item{\code{names}: As for \code{$import} +##' \item{\code{names}: As for `$import` ##' } ##' -##' \item{\code{namespace}: Namespace to import objects into. If \code{NULL}, then imports all namespaces from the source storr. +##' \item{\code{namespace}: Namespace to import objects into. If `NULL`, then imports all namespaces from the source storr. ##' } ##' } ##' } ##' \item{\code{index_export}}{ -##' Generate a data.frame with an index of objects present in a storr. This can be saved (for an rds storr) in lieu of the keys/ directory and re-imported with \code{index_import}. It will provide a more version control friendly export of the data in a storr. +##' Generate a data.frame with an index of objects present in a storr. This can be saved (for an rds storr) in lieu of the keys/ directory and re-imported with `index_import`. It will provide a more version control friendly export of the data in a storr. ##' ##' \emph{Usage:} ##' \code{index_export(namespace = NULL)} diff --git a/man/driver_remote.Rd b/man/driver_remote.Rd index dae2427..785c904 100644 --- a/man/driver_remote.Rd +++ b/man/driver_remote.Rd @@ -10,9 +10,8 @@ driver_remote(ops, ..., path_local = NULL) \item{ops}{A file operations object. See tests for now to see what is required to implement one.} -\item{...}{Arguments to pass through to \code{\link{driver_rds}}, -including \code{compress}, \code{mangle_key}, -\code{mangle_key_pad} and \code{hash_algorithm}.} +\item{...}{Arguments to pass through to \link{driver_rds}, including +\code{compress}, \code{mangle_key}, \code{mangle_key_pad} and \code{hash_algorithm}.} \item{path_local}{Path to a local cache. This can be left as \code{NULL}, in which case a per-session cache will be used. @@ -30,7 +29,7 @@ so there is no \code{storr_remote} function. Instead this function is designed to support external packages that implement the details. For a worked example, see the package tests (\code{helper-remote.R}). In the current implementation these -build off of the \code{\link{driver_rds}} driver by copying files +build off of the \link{driver_rds} driver by copying files to some remote location. } \author{ diff --git a/man/fetch_hook_read.Rd b/man/fetch_hook_read.Rd index e0cd966..ff38755 100644 --- a/man/fetch_hook_read.Rd +++ b/man/fetch_hook_read.Rd @@ -7,11 +7,11 @@ fetch_hook_read(fpath, fread) } \arguments{ -\item{fpath}{Function to convert \code{key, namespace} into a file +\item{fpath}{Function to convert \verb{key, namespace} into a file path} \item{fread}{Function for converting \code{filename} into an R -pobject} +object} } \description{ Hook to fetch a resource from a file, for use with @@ -23,7 +23,7 @@ remote resources. } \details{ For more information about using this, see -\code{\link{storr_external}} (this can be used as a +\link{storr_external} (this can be used as a \code{fetch_hook} argument) and the vignette: \code{vignette("external")} } diff --git a/man/storr-defunct.Rd b/man/storr-defunct.Rd index 7998c5d..85e35f1 100644 --- a/man/storr-defunct.Rd +++ b/man/storr-defunct.Rd @@ -16,12 +16,12 @@ storr_redis_api(...) Defunct functions } \details{ -The redis functions (\code{driver_redis_api} and -\code{storr_redis_api}) have been moved out of this package and -into Redis. I don't believe anyone is using them at the time of -the move so this is being done fairly abruptly - this is -unfortunate, but necessary to avoid a circular dependency! The -new functions are simply \code{redux::driver_redis_api} and -\code{redux::storr_redis_api}, along with a helper function -\code{redux::storr_hiredis} which also creates the connection. +The Redis functions (\code{driver_redis_api} and \code{storr_redis_api}) +have been moved out of this package and into \code{redux}. I don't +believe anyone is using them at the time of the move so this is +being done fairly abruptly - this is unfortunate, but necessary to +avoid a circular dependency! The new functions are simply +\code{redux::driver_redis_api} and \code{redux::storr_redis_api}, along with +a helper function \code{redux::storr_hiredis} which also creates the +connection. } diff --git a/man/storr.Rd b/man/storr.Rd index 4d92b76..4a3dc1e 100644 --- a/man/storr.Rd +++ b/man/storr.Rd @@ -10,38 +10,38 @@ storr(driver, default_namespace = "objects") \item{driver}{A driver object} \item{default_namespace}{Default namespace to store objects in. -By default \code{"objects"} is used, but this might be useful to -have two diffent \code{storr} objects pointing at the same +By default \code{"objects"} is used, but this might be useful to have +two different \code{storr} objects pointing at the same underlying storage, but storing things in different namespaces.} } \description{ Create an object cache; a "storr". A storr is a simple key-value -store where the actual content is stored in a content-addressible +store where the actual content is stored in a content-addressable way (so that duplicate objects are only stored once) and with a caching layer so that repeated lookups are fast even if the underlying storage driver is slow. } \details{ To create a storr you need to provide a "driver" object. There -are three in this package: \code{\link{driver_environment}} for -ephemeral in-memory storage, \code{\link{driver_rds}} for -serialized storage to disk, and \code{\link{driver_dbi}} for use -with DBI-compliant database interfaces. The \code{redux} package -(on CRAN) provides a storr driver that uses Redis. +are three in this package: \link{driver_environment} for ephemeral +in-memory storage, \link{driver_rds} for serialized storage to disk, +and \link{driver_dbi} for use with DBI-compliant database interfaces. +The \code{redux} package (on CRAN) provides a storr driver that uses +Redis. -There are convenience functions (e.g., -\code{\link{storr_environment}} and \code{\link{storr_rds}}) that -may be more convenient to use than this function. +There are convenience functions (e.g., \link{storr_environment} and +\link{storr_rds}) that may be more convenient to use than this +function. Once a storr has been made it provides a number of methods. -Because storr uses \code{R6} (\code{\link{R6Class}}) objects, each +Because storr uses \code{R6} (\link[R6:R6Class]{R6::R6Class}) objects, each method is accessed by using \code{$} on a storr object (see the examples). The methods are described below in the "Methods" section. -The \code{default_namespace} affects all methods of the storr -object that refer to namespaces; if a namespace is not given, then -the action (get, set, del, list, import, export) will affect the +The \code{default_namespace} affects all methods of the storr object +that refer to namespaces; if a namespace is not given, then the +action (get, set, del, list, import, export) will affect the \code{default_namespace}. By default this is \code{"objects"}. } \section{Methods}{ @@ -49,464 +49,500 @@ the action (get, set, del, list, import, export) will affect the \describe{ \item{\code{destroy}}{ - Totally destroys the storr by telling the driver to destroy all the data and then deleting the driver. This will remove all data and cannot be undone. +Totally destroys the storr by telling the driver to destroy all the data and then deleting the driver. This will remove all data and cannot be undone. - \emph{Usage:} - \code{destroy()} +\emph{Usage:} +\code{destroy()} } \item{\code{flush_cache}}{ - Flush the temporary cache of objects that accumulates as the storr is used. Should not need to be called often. +Flush the temporary cache of objects that accumulates as the storr is used. Should not need to be called often. - \emph{Usage:} - \code{flush_cache()} +\emph{Usage:} +\code{flush_cache()} } \item{\code{set}}{ - Set a key to a value. +Set a key to a value. - \emph{Usage:} - \code{set(key, value, namespace = self$default_namespace, use_cache = TRUE)} +\emph{Usage:} +\code{set(key, value, namespace = self$default_namespace, use_cache = TRUE)} - \emph{Arguments:} - \itemize{ - \item{\code{key}: The key name. Can be any string. - } +\emph{Arguments:} +\itemize{ +\item{\code{key}: The key name. Can be any string. +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{value}: Any R object to store. The object will generally be serialized (this is not actually true for the environment storr) so only objects that would usually be expected to survive a `saveRDS`/`readRDS` roundtrip will work. This excludes Rcpp modules objects, external pointers, etc. But any "normal" R object will work fine. +\} - \item{\code{value}: Any R object to store. The object will generally be serialized (this is not actually true for the environment storr) so only objects that would usually be expected to survive a \code{saveRDS}/\code{readRDS} roundtrip will work. This excludes Rcpp modules objects, external pointers, etc. But any "normal" R object will work fine. - } +\\item\{\code{namespace}: An optional namespace. By default the default namespace that the storr was created with will be used (by default that is "objects"). Different namespaces allow different types of objects to be stored without risk of names colliding. Use of namespaces is optional, but if used they must be a string. +\} - \item{\code{namespace}: An optional namespace. By default the default namespace that the storr was created with will be used (by default that is "objects"). Different namespaces allow different types of objects to be stored without risk of names colliding. Use of namespaces is optional, but if used they must be a string. - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} - \emph{Value}: - Invisibly, the hash of the saved object. +\emph{Value}: +Invisibly, the hash of the saved object. } \item{\code{set_by_value}}{ - Like \code{set} but saves the object with a key that is the same as the hash of the object. Equivalent to \code{$set(digest::digest(value), value)}. +Like \code{set} but saves the object with a key that is the same as the hash of the object. Equivalent to \verb{$set(digest::digest(value), value)}. + +\emph{Usage:} +\code{set_by_value(value, namespace = self$default_namespace, use_cache = TRUE)} - \emph{Usage:} - \code{set_by_value(value, namespace = self$default_namespace, use_cache = TRUE)} +\emph{Arguments:} +\itemize{ +\item{\code{value}: An R object to save, with the same limitations as \code{set}. +} - \emph{Arguments:} - \itemize{ - \item{\code{value}: An R object to save, with the same limitations as \code{set}. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: Optional namespace to save the key into. +\} - \item{\code{namespace}: Optional namespace to save the key into. - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} } \item{\code{get}}{ - Retrieve an object from the storr. If the requested value is not found then a \code{KeyError} will be raised (an R error, but can be caught with \code{tryCatch}; see the "storr" vignette). +Retrieve an object from the storr. If the requested value is not found then a \code{KeyError} will be raised (an R error, but can be caught with \code{tryCatch}; see the "storr" vignette). - \emph{Usage:} - \code{get(key, namespace = self$default_namespace, use_cache = TRUE)} +\emph{Usage:} +\code{get(key, namespace = self$default_namespace, use_cache = TRUE)} + +\emph{Arguments:} +\itemize{ +\item{\code{key}: The name of the key to get. +} - \emph{Arguments:} - \itemize{ - \item{\code{key}: The name of the key to get. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: Optional namespace to look for the key within. +\} - \item{\code{namespace}: Optional namespace to look for the key within. - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} } \item{\code{get_hash}}{ - Retrieve the hash of an object stored in the storr (rather than the object itself). +Retrieve the hash of an object stored in the storr (rather than the object itself). + +\emph{Usage:} +\code{get_hash(key, namespace = self$default_namespace)} - \emph{Usage:} - \code{get_hash(key, namespace = self$default_namespace)} +\emph{Arguments:} +\itemize{ +\item{\code{key}: The name of the key to get. +} - \emph{Arguments:} - \itemize{ - \item{\code{key}: The name of the key to get. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: Optional namespace to look for the key within. +\} +}\if{html}{\out{
}} - \item{\code{namespace}: Optional namespace to look for the key within. - } - } +} } \item{\code{del}}{ - Delete an object fom the storr. +Delete an object from the storr. - \emph{Usage:} - \code{del(key, namespace = self$default_namespace)} +\emph{Usage:} +\code{del(key, namespace = self$default_namespace)} - \emph{Arguments:} - \itemize{ - \item{\code{key}: A vector of names of keys - } +\emph{Arguments:} +\itemize{ +\item{\code{key}: A vector of names of keys +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: The namespace of the key. +\} +}\if{html}{\out{
}} - \item{\code{namespace}: The namespace of the key. - } - } +} - \emph{Value}: - A logical vector the same length as the recycled length of key/namespace, with each element being \code{TRUE} if an object was deleted, \code{FALSE} otherwise. +\emph{Value}: +A logical vector the same length as the recycled length of key/namespace, with each element being \code{TRUE} if an object was deleted, \code{FALSE} otherwise. } \item{\code{duplicate}}{ - Duplicate the value of a set of keys into a second set of keys. Because the value stored against a key is just the hash of its content, this operation is very efficient - it does not make a copy of the data, just the pointer to the data (for more details see the storr vignette which explains the storage model in more detail). Multiple keys (and/or namespaces) can be provided, with keys and nmespaces recycled as needed. However, the number of source and destination keys must be the same. The order of operation is not defined, so if the sets of keys are overlapping it is undefined behaviour. +Duplicate the value of a set of keys into a second set of keys. Because the value stored against a key is just the hash of its content, this operation is very efficient - it does not make a copy of the data, just the pointer to the data (for more details see the storr vignette which explains the storage model in more detail). Multiple keys (and/or namespaces) can be provided, with keys and namespaces recycled as needed. However, the number of source and destination keys must be the same. The order of operation is not defined, so if the sets of keys are overlapping it is undefined behaviour. - \emph{Usage:} - \code{duplicate(key_src, key_dest, namespace = self$default_namespace, +\emph{Usage:} +\code{duplicate(key_src, key_dest, namespace = self$default_namespace, namespace_src = namespace, namespace_dest = namespace)} - \emph{Arguments:} - \itemize{ - \item{\code{key_src}: The source key (or vector of keys) - } +\emph{Arguments:} +\itemize{ +\item{\code{key_src}: The source key (or vector of keys) +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{key_dest}: The destination key +\} - \item{\code{key_dest}: The destination key - } +\\item\{\code{namespace}: The namespace to copy keys within (used only of `namespace_src` and `namespace_dest` are not provided +\} - \item{\code{namespace}: The namespace to copy keys within (used only of \code{namespace_src} and \code{namespace_dest} are not provided - } +\\item\{\code{namespace_src}: The source namespace - use this where keys are duplicated across namespaces. +\} - \item{\code{namespace_src}: The source namespace - use this where keys are duplicated across namespaces. - } +\\item\{\code{namespace_dest}: The destination namespace - use this where keys are duplicated across namespaces. +\} +}\if{html}{\out{
}} - \item{\code{namespace_dest}: The destination namespace - use this where keys are duplicated across namespaces. - } - } +} } \item{\code{fill}}{ - Set one or more keys (potentially across namespaces) to the same value, without duplication effort serialisation, or duplicating data. +Set one or more keys (potentially across namespaces) to the same value, without duplication effort serialisation, or duplicating data. - \emph{Usage:} - \code{fill(key, value, namespace = self$default_namespace, use_cache = TRUE)} +\emph{Usage:} +\code{fill(key, value, namespace = self$default_namespace, use_cache = TRUE)} + +\emph{Arguments:} +\itemize{ +\item{\code{key}: A vector of keys to get; zero to many valid keys +} - \emph{Arguments:} - \itemize{ - \item{\code{key}: A vector of keys to get; zero to many valid keys - } +\if{html}{\out{
}}\preformatted{\\item\{\code{value}: A single value to set all keys to +\} - \item{\code{value}: A single value to set all keys to - } +\\item\{\code{namespace}: A vector of namespaces (either a single namespace or a vector) +\} - \item{\code{namespace}: A vector of namespaces (either a single namespace or a vector) - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} } \item{\code{clear}}{ - Clear a storr. This function might be slow as it will iterate over each key. Future versions of storr might allow drivers to implement a bulk clear method that will allow faster clearing. +Clear a storr. This function might be slow as it will iterate over each key. Future versions of storr might allow drivers to implement a bulk clear method that will allow faster clearing. - \emph{Usage:} - \code{clear(namespace = self$default_namespace)} +\emph{Usage:} +\code{clear(namespace = self$default_namespace)} - \emph{Arguments:} - \itemize{ - \item{\code{namespace}: A namespace, to clear a single namespace, or \code{NULL} to clear all namespaces. - } - } +\emph{Arguments:} +\itemize{ +\item{\code{namespace}: A namespace, to clear a single namespace, or \code{NULL} to clear all namespaces. +} +} } \item{\code{exists}}{ - Test if a key exists within a namespace +Test if a key exists within a namespace + +\emph{Usage:} +\code{exists(key, namespace = self$default_namespace)} - \emph{Usage:} - \code{exists(key, namespace = self$default_namespace)} +\emph{Arguments:} +\itemize{ +\item{\code{key}: A vector of names of keys +} - \emph{Arguments:} - \itemize{ - \item{\code{key}: A vector of names of keys - } +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: The namespace of the key. +\} +}\if{html}{\out{
}} - \item{\code{namespace}: The namespace of the key. - } - } +} - \emph{Value}: - A logical vector the same length as the recycled length of key/namespace, with each element being \code{TRUE} if the object exists and \code{FALSE} otherwise. +\emph{Value}: +A logical vector the same length as the recycled length of key/namespace, with each element being \code{TRUE} if the object exists and \code{FALSE} otherwise. } \item{\code{exists_object}}{ - Test if an object with a given hash exists within the storr +Test if an object with a given hash exists within the storr - \emph{Usage:} - \code{exists_object(hash)} +\emph{Usage:} +\code{exists_object(hash)} - \emph{Arguments:} - \itemize{ - \item{\code{hash}: Hash to test - } - } +\emph{Arguments:} +\itemize{ +\item{\code{hash}: Hash to test +} +} } \item{\code{mset}}{ - Set multiple elements at once +Set multiple elements at once - \emph{Usage:} - \code{mset(key, value, namespace = self$default_namespace, use_cache = TRUE)} +\emph{Usage:} +\code{mset(key, value, namespace = self$default_namespace, use_cache = TRUE)} - \emph{Arguments:} - \itemize{ - \item{\code{key}: A vector of keys to set; zero to many valid keys - } +\emph{Arguments:} +\itemize{ +\item{\code{key}: A vector of keys to set; zero to many valid keys +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{value}: A vector of values +\} - \item{\code{value}: A vector of values - } +\\item\{\code{namespace}: A vector of namespaces (either a single namespace or a vector) +\} - \item{\code{namespace}: A vector of namespaces (either a single namespace or a vector) - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} - \emph{Details:} - The arguments \code{key} and \code{namespace} are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. +\emph{Details:} +The arguments \code{key} and \code{namespace} are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. } \item{\code{mget}}{ - Get multiple elements at once +Get multiple elements at once - \emph{Usage:} - \code{mget(key, namespace = self$default_namespace, use_cache = TRUE, +\emph{Usage:} +\code{mget(key, namespace = self$default_namespace, use_cache = TRUE, missing = NULL)} - \emph{Arguments:} - \itemize{ - \item{\code{key}: A vector of keys to get; zero to many valid keys - } +\emph{Arguments:} +\itemize{ +\item{\code{key}: A vector of keys to get; zero to many valid keys +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: A vector of namespaces (either a single namespace or a vector) +\} - \item{\code{namespace}: A vector of namespaces (either a single namespace or a vector) - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } +\\item\{\code{missing}: Value to use for missing elements; by default `NULL` will be used. IF `NULL` is a value that you might have stored in the storr you might want to use a different value here to distinguish "missing" from "set to NULL". In addition, the `missing` attribute will indicate which values were missing. +\} +}\if{html}{\out{
}} - \item{\code{missing}: Value to use for missing elements; by default \code{NULL} will be used. IF \code{NULL} is a value that you might have stored in the storr you might want to use a different value here to distinguish "missing" from "set to NULL". In addition, the \code{missing} attribute will indicate which values were missing. - } - } +} - \emph{Details:} - The arguments \code{key} and \code{namespace} are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. +\emph{Details:} +The arguments \code{key} and \code{namespace} are recycled such that either can be given as a scalar if the other is a vector. Other recycling is not allowed. - \emph{Value}: - A list with a length of the recycled length of \code{key} and \code{namespace}. If any elements are missing, then an attribute \code{missing} will indicate the elements that are missing (this will be an integer vector with the indices of values were not found in the storr). +\emph{Value}: +A list with a length of the recycled length of \code{key} and \code{namespace}. If any elements are missing, then an attribute \code{missing} will indicate the elements that are missing (this will be an integer vector with the indices of values were not found in the storr). } \item{\code{mset_by_value}}{ - Set multiple elements at once, by value. A cross between \code{mset} and \code{set_by_value}. +Set multiple elements at once, by value. A cross between \code{mset} and \code{set_by_value}. - \emph{Usage:} - \code{mset_by_value(value, namespace = self$default_namespace, use_cache = TRUE)} +\emph{Usage:} +\code{mset_by_value(value, namespace = self$default_namespace, use_cache = TRUE)} + +\emph{Arguments:} +\itemize{ +\item{\code{value}: A list or vector of values to set into the storr. +} - \emph{Arguments:} - \itemize{ - \item{\code{value}: A list or vector of values to set into the storr. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{namespace}: A vector of namespaces +\} - \item{\code{namespace}: A vector of namespaces - } +\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} } \item{\code{gc}}{ - Garbage collect the storr. Because keys do not directly map to objects, but instead map to hashes which map to objects, it is possible that hash/object pairs can persist with nothing pointing at them. Running \code{gc} will remove these objects from the storr. +Garbage collect the storr. Because keys do not directly map to objects, but instead map to hashes which map to objects, it is possible that hash/object pairs can persist with nothing pointing at them. Running \code{gc} will remove these objects from the storr. - \emph{Usage:} - \code{gc()} +\emph{Usage:} +\code{gc()} } \item{\code{get_value}}{ - Get the content of an object given its hash. +Get the content of an object given its hash. + +\emph{Usage:} +\code{get_value(hash, use_cache = TRUE)} - \emph{Usage:} - \code{get_value(hash, use_cache = TRUE)} +\emph{Arguments:} +\itemize{ +\item{\code{hash}: The hash of the object to retrieve. +} - \emph{Arguments:} - \itemize{ - \item{\code{hash}: The hash of the object to retrieve. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} - \emph{Value}: - The object if it is present, otherwise throw a \code{HashError}. +\emph{Value}: +The object if it is present, otherwise throw a \code{HashError}. } \item{\code{set_value}}{ - Add an object value, but don't add a key. You will not need to use this very often, but it is used internally. +Add an object value, but don't add a key. You will not need to use this very often, but it is used internally. - \emph{Usage:} - \code{set_value(value, use_cache = TRUE)} +\emph{Usage:} +\code{set_value(value, use_cache = TRUE)} - \emph{Arguments:} - \itemize{ - \item{\code{value}: An R object to set. - } +\emph{Arguments:} +\itemize{ +\item{\code{value}: An R object to set. +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} - \emph{Value}: - Invisibly, the hash of the object. +\emph{Value}: +Invisibly, the hash of the object. } \item{\code{mset_value}}{ - Add a vector of object values, but don't add keys. You will not need to use this very often, but it is used internally. +Add a vector of object values, but don't add keys. You will not need to use this very often, but it is used internally. + +\emph{Usage:} +\code{mset_value(values, use_cache = TRUE)} - \emph{Usage:} - \code{mset_value(values, use_cache = TRUE)} +\emph{Arguments:} +\itemize{ +\item{\code{values}: A list of R objects to set +} - \emph{Arguments:} - \itemize{ - \item{\code{values}: A list of R objects to set - } +\if{html}{\out{
}}\preformatted{\\item\{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). +\} +}\if{html}{\out{
}} - \item{\code{use_cache}: Use the internal cache to avoid reading or writing to the underlying storage if the data has already been seen (i.e., we have seen the hash of the object before). - } - } +} } \item{\code{list}}{ - List all keys stored in a namespace. +List all keys stored in a namespace. - \emph{Usage:} - \code{list(namespace = self$default_namespace)} +\emph{Usage:} +\code{list(namespace = self$default_namespace)} - \emph{Arguments:} - \itemize{ - \item{\code{namespace}: The namespace to list keys within. - } - } +\emph{Arguments:} +\itemize{ +\item{\code{namespace}: The namespace to list keys within. +} +} - \emph{Value}: - A sorted character vector (possibly zero-length). +\emph{Value}: +A sorted character vector (possibly zero-length). } \item{\code{list_hashes}}{ - List all hashes stored in the storr +List all hashes stored in the storr - \emph{Usage:} - \code{list_hashes()} +\emph{Usage:} +\code{list_hashes()} - \emph{Value}: - A sorted character vector (possibly zero-length). +\emph{Value}: +A sorted character vector (possibly zero-length). } \item{\code{list_namespaces}}{ - List all namespaces known to the database +List all namespaces known to the database - \emph{Usage:} - \code{list_namespaces()} +\emph{Usage:} +\code{list_namespaces()} - \emph{Value}: - A sorted character vector (possibly zero-length). +\emph{Value}: +A sorted character vector (possibly zero-length). } \item{\code{import}}{ - Import R objects from an environment. +Import R objects from an environment. - \emph{Usage:} - \code{import(src, list = NULL, namespace = self$default_namespace, +\emph{Usage:} +\code{import(src, list = NULL, namespace = self$default_namespace, skip_missing = FALSE)} - \emph{Arguments:} - \itemize{ - \item{\code{src}: Object to import objects from; can be a list, environment or another storr. - } +\emph{Arguments:} +\itemize{ +\item{\code{src}: Object to import objects from; can be a list, environment or another storr. +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{list}: Names of of objects to import (or `NULL` to import all objects in `envir`. If given it must be a character vector. If named, the names of the character vector will be the names of the objects as created in the storr. +\} - \item{\code{list}: Names of of objects to import (or \code{NULL} to import all objects in \code{envir}. If given it must be a character vector. If named, the names of the character vector will be the names of the objects as created in the storr. - } +\\item\{\code{namespace}: Namespace to get objects from, and to put objects into. If `NULL`, all namespaces from `src` will be imported. If named, then the same rule is followed as `list`; `namespace = c(a = b)` will import the contents of namespace `b` as namespace `a`. +\} - \item{\code{namespace}: Namespace to get objects from, and to put objects into. If \code{NULL}, all namespaces from \code{src} will be imported. If named, then the same rule is followed as \code{list}; \code{namespace = c(a = b)} will import the contents of namespace \code{b} as namespace \code{a}. - } +\\item\{\code{skip_missing}: Logical, indicating if missing keys (specified in `list`) should be skipped over, rather than being treated as an error (the default). +\} +}\if{html}{\out{
}} - \item{\code{skip_missing}: Logical, indicating if missing keys (specified in \code{list}) should be skipped over, rather than being treated as an error (the default). - } - } +} } \item{\code{export}}{ - Export objects from the storr into something else. +Export objects from the storr into something else. - \emph{Usage:} - \code{export(dest, list = NULL, namespace = self$default_namespace, +\emph{Usage:} +\code{export(dest, list = NULL, namespace = self$default_namespace, skip_missing = FALSE)} - \emph{Arguments:} - \itemize{ - \item{\code{dest}: A target destination to export objects to; can be a list, environment, or another storr. Use \code{list()} to export to a brand new list, or use \code{as.list(object)} for a shorthand. - } +\emph{Arguments:} +\itemize{ +\item{\code{dest}: A target destination to export objects to; can be a list, environment, or another storr. Use \code{list()} to export to a brand new list, or use \code{as.list(object)} for a shorthand. +} + +\if{html}{\out{
}}\preformatted{\\item\{\code{list}: Names of objects to export, with the same rules as `list` in `$import`. +\} - \item{\code{list}: Names of objects to export, with the same rules as \code{list} in \code{$import}. - } +\\item\{\code{namespace}: Namespace to get objects from, and to put objects into. If `NULL`, then this will export namespaces from this (source) storr into the destination; if there is more than one namespace,this is only possible if `dest` is a storr (otherwise there will be an error). +\} - \item{\code{namespace}: Namespace to get objects from, and to put objects into. If \code{NULL}, then this will export namespaces from this (source) storr into the destination; if there is more than one namespace,this is only possible if \code{dest} is a storr (otherwise there will be an error). - } +\\item\{\code{skip_missing}: Logical, indicating if missing keys (specified in `list`) should be skipped over, rather than being treated as an error (the default). +\} +}\if{html}{\out{
}} - \item{\code{skip_missing}: Logical, indicating if missing keys (specified in \code{list}) should be skipped over, rather than being treated as an error (the default). - } - } +} - \emph{Value}: - Invisibly, \code{dest}, which allows use of \code{e <- st$export(new.env())} and \code{x <- st$export(list())}. +\emph{Value}: +Invisibly, \code{dest}, which allows use of \code{e <- st$export(new.env())} and \code{x <- st$export(list())}. } \item{\code{archive_export}}{ - Export objects from the storr into a special "archive" storr, which is an \code{\link{storr_rds}} with name mangling turned on (which encodes keys with base64 so that they do not voilate filesystem naming conventions). +Export objects from the storr into a special "archive" storr, which is an \link{storr_rds} with name mangling turned on (which encodes keys with base64 so that they do not violate filesystem naming conventions). - \emph{Usage:} - \code{archive_export(path, names = NULL, namespace = NULL)} +\emph{Usage:} +\code{archive_export(path, names = NULL, namespace = NULL)} + +\emph{Arguments:} +\itemize{ +\item{\code{path}: Path to create the storr at; can exist already. +} - \emph{Arguments:} - \itemize{ - \item{\code{path}: Path to create the storr at; can exist already. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{names}: As for `$export` +\} - \item{\code{names}: As for \code{$export} - } +\\item\{\code{namespace}: Namespace to get objects from. If `NULL`, then exports all namespaces found in this (source) storr. +\} +}\if{html}{\out{
}} - \item{\code{namespace}: Namespace to get objects from. If \code{NULL}, then exports all namespaces found in this (source) storr. - } - } +} } \item{\code{archive_import}}{ - Inverse of \code{archive_export}; import objects from a storr that was created by \code{archive_export}. +Inverse of \code{archive_export}; import objects from a storr that was created by \code{archive_export}. + +\emph{Usage:} +\code{archive_import(path, names = NULL, namespace = NULL)} - \emph{Usage:} - \code{archive_import(path, names = NULL, namespace = NULL)} +\emph{Arguments:} +\itemize{ +\item{\code{path}: Path of the exported storr. +} - \emph{Arguments:} - \itemize{ - \item{\code{path}: Path of the exported storr. - } +\if{html}{\out{
}}\preformatted{\\item\{\code{names}: As for `$import` +\} - \item{\code{names}: As for \code{$import} - } +\\item\{\code{namespace}: Namespace to import objects into. If `NULL`, then imports all namespaces from the source storr. +\} +}\if{html}{\out{
}} - \item{\code{namespace}: Namespace to import objects into. If \code{NULL}, then imports all namespaces from the source storr. - } - } +} } \item{\code{index_export}}{ - Generate a data.frame with an index of objects present in a storr. This can be saved (for an rds storr) in lieu of the keys/ directory and re-imported with \code{index_import}. It will provide a more version control friendly export of the data in a storr. +Generate a data.frame with an index of objects present in a storr. This can be saved (for an rds storr) in lieu of the keys/ directory and re-imported with \code{index_import}. It will provide a more version control friendly export of the data in a storr. - \emph{Usage:} - \code{index_export(namespace = NULL)} +\emph{Usage:} +\code{index_export(namespace = NULL)} - \emph{Arguments:} - \itemize{ - \item{\code{namespace}: Optional character vector of namespaces to export. The default is to export all namespaces. - } - } +\emph{Arguments:} +\itemize{ +\item{\code{namespace}: Optional character vector of namespaces to export. The default is to export all namespaces. +} +} } \item{\code{index_import}}{ - Import an index. +Import an index. - \emph{Usage:} - \code{index_import(index)} +\emph{Usage:} +\code{index_import(index)} - \emph{Arguments:} - \itemize{ - \item{\code{index}: Must be a data.frame with columns 'namespace', 'key' and 'hash' (in any order). It is an error if not all hashes are present in the storr. - } - } +\emph{Arguments:} +\itemize{ +\item{\code{index}: Must be a data.frame with columns 'namespace', 'key' and 'hash' (in any order). It is an error if not all hashes are present in the storr. +} +} } } } @@ -533,7 +569,7 @@ st$list_hashes() st$del("mykey") ## Storr objects can be created that have a default namespace that is -## not "objects" by using the \\code{default_namespace} argument (this +## not "objects" by using the `default_namespace` argument (this ## one also points at the same memory as the first storr). st2 <- storr(driver_environment(st$driver$envir), default_namespace = "namespace2") diff --git a/man/storr_dbi.Rd b/man/storr_dbi.Rd index efba7b3..6f7fc13 100644 --- a/man/storr_dbi.Rd +++ b/man/storr_dbi.Rd @@ -5,11 +5,24 @@ \alias{driver_dbi} \title{DBI storr driver} \usage{ -storr_dbi(tbl_data, tbl_keys, con, args = NULL, binary = NULL, - hash_algorithm = NULL, default_namespace = "objects") - -driver_dbi(tbl_data, tbl_keys, con, args = NULL, binary = NULL, - hash_algorithm = NULL) +storr_dbi( + tbl_data, + tbl_keys, + con, + args = NULL, + binary = NULL, + hash_algorithm = NULL, + default_namespace = "objects" +) + +driver_dbi( + tbl_data, + tbl_keys, + con, + args = NULL, + binary = NULL, + hash_algorithm = NULL +) } \arguments{ \item{tbl_data}{Name for the table that maps hashes to values} @@ -24,7 +37,7 @@ driver_dbi(tbl_data, tbl_keys, con, args = NULL, binary = NULL, \item{binary}{Optional logical indicating if the values should be stored in binary. If possible, this is both (potentially faster) and more accurate. However, at present it is supported -only under very recent DBI and RSQLite packages, and for no +only under very recent \code{DBI} and \code{RSQLite} packages, and for no other DBI drivers, and is not actually any faster. If not given (i.e., \code{NULL}), then binary storage will be used where possible when creating new tables, and where tables exist, we @@ -32,16 +45,15 @@ use whatever was used in the existing tables.} \item{hash_algorithm}{Name of the hash algorithm to use. Possible values are "md5", "sha1", and others supported by -\code{\link{digest}}. If not given, then we will default to +\link[digest:digest]{digest::digest}. If not given, then we will default to "md5".} -\item{default_namespace}{Default namespace (see -\code{\link{storr}}).} +\item{default_namespace}{Default namespace (see \link{storr}).} } \description{ Object cache driver using the "DBI" package interface for storage. This means that storr can work for any supported "DBI" driver -(though practically this works only for SQLite and Postgres until +(though practically this works only for \code{SQLite} and Postgres until some MySQL dialect translation is done). To connect, you must provide the \emph{driver} object (e.g., \code{RSQLite::SQLite()}, or \code{RPostgres::Postgres()} as the first argument. @@ -59,7 +71,7 @@ treated as opaque (don't use them for anything else - reading or writing). Apart from that the names do not matter. Because of treatment of binary data by the underlying DBI drivers, -binary serialistion is not any faster (and might be slightly +binary serialisation is not any faster (and might be slightly slower than) string serialisation, in contrast with my experience with other backends. @@ -70,7 +82,7 @@ can't be interpolated in the same way - these storr simply quotes, but validates beforehand to ensure that \code{tbl_data} and \code{tbl_keys} do not contain quotes. -Be aware that \code{$destroy()} will close the connection to the +Be aware that \verb{$destroy()} will close the connection to the database. } \examples{ diff --git a/man/storr_environment.Rd b/man/storr_environment.Rd index 101c346..ff1a1cf 100644 --- a/man/storr_environment.Rd +++ b/man/storr_environment.Rd @@ -5,8 +5,11 @@ \alias{driver_environment} \title{Environment object cache driver} \usage{ -storr_environment(envir = NULL, hash_algorithm = NULL, - default_namespace = "objects") +storr_environment( + envir = NULL, + hash_algorithm = NULL, + default_namespace = "objects" +) driver_environment(envir = NULL, hash_algorithm = NULL) } @@ -19,15 +22,15 @@ argument along.} \item{hash_algorithm}{Name of the hash algorithm to use. Possible values are "md5", "sha1", and others supported by -\code{\link{digest}}. If not given, then we will default to +\link[digest:digest]{digest::digest}. If not given, then we will default to "md5".} -\item{default_namespace}{Default namespace (see \code{\link{storr}}).} +\item{default_namespace}{Default namespace (see \link{storr}).} } \description{ Fast but transient environment driver. This driver saves objects in a local R environment, without serialisation. This makes -lookup fast but it cannot be saved across sesssions. The +lookup fast but it cannot be saved across sessions. The environment storr can be made persistent by saving it out as a file storr though. } diff --git a/man/storr_external.Rd b/man/storr_external.Rd index 10e77c2..0b74fcb 100644 --- a/man/storr_external.Rd +++ b/man/storr_external.Rd @@ -16,7 +16,7 @@ not found in the store. This function must take arguments throw an error if the external resource cannot be resolved.} \item{default_namespace}{Default namespace (see -\code{\link{storr}})} +\link{storr})} } \description{ storr for fetching external resources. This driver is used where diff --git a/man/storr_multistorr.Rd b/man/storr_multistorr.Rd index 001105b..92fcc9e 100644 --- a/man/storr_multistorr.Rd +++ b/man/storr_multistorr.Rd @@ -11,8 +11,7 @@ storr_multistorr(keys, data, default_namespace = "objects") \item{data}{Driver for the data} -\item{default_namespace}{Default namespace (see -\code{\link{storr}}).} +\item{default_namespace}{Default namespace (see \link{storr})} } \description{ Create a special storr that uses separate storage drivers for the diff --git a/man/storr_rds.Rd b/man/storr_rds.Rd index a5c4e23..b8ea056 100644 --- a/man/storr_rds.Rd +++ b/man/storr_rds.Rd @@ -5,12 +5,22 @@ \alias{driver_rds} \title{rds object cache driver} \usage{ -storr_rds(path, compress = NULL, mangle_key = NULL, - mangle_key_pad = NULL, hash_algorithm = NULL, - default_namespace = "objects") - -driver_rds(path, compress = NULL, mangle_key = NULL, - mangle_key_pad = NULL, hash_algorithm = NULL) +storr_rds( + path, + compress = NULL, + mangle_key = NULL, + mangle_key_pad = NULL, + hash_algorithm = NULL, + default_namespace = "objects" +) + +driver_rds( + path, + compress = NULL, + mangle_key = NULL, + mangle_key_pad = NULL, + hash_algorithm = NULL +) } \arguments{ \item{path}{Path for the store. \code{tempdir()} is a good choice @@ -24,25 +34,23 @@ amount of space for a reasonable amount of time.} using base64 before saving to the filesystem. See Details.} \item{mangle_key_pad}{Logical indicating if the filenames created -when using \code{mangle_key} should also be "padded" with the -\code{=} character to make up a round number of bytes. Padding -is required to satisfy the document that describes base64 -encoding (RFC 4648) but can cause problems in some applications -(see \href{https://github.com/richfitz/storr/issues/43}{this -issue}. The default is to not pad \emph{new} storr archives. -This should be generally safe to leave alone.} +when using \code{mangle_key} should also be "padded" with the \code{=} +character to make up a round number of bytes. Padding is +required to satisfy the document that describes base64 encoding +(RFC 4648) but can cause problems in some applications (see +\href{https://github.com/richfitz/storr/issues/43}{this issue}). The +default is to not pad \emph{new} storr archives. This should be +generally safe to leave alone.} \item{hash_algorithm}{Name of the hash algorithm to use. Possible values are "md5", "sha1", and others supported by -\code{\link{digest}}. If not given, then we will default to -"md5".} +\link[digest:digest]{digest::digest}. If not given, then we will default to "md5".} -\item{default_namespace}{Default namespace (see -\code{\link{storr}}).} +\item{default_namespace}{Default namespace (see \link{storr}).} } \description{ Object cache driver that saves objects using R's native -serialized file format (see \code{\link{saveRDS}}) on the +serialized file format (see \link{saveRDS}) on the filesystem. } \details{ @@ -56,7 +64,7 @@ transparent to the user -- the storr will appear to store things with unmangled keys but the names of the stored files will be different. -Note that the \emph{namespace} is not mangled (at least not yet) +Note that the (\emph{namespace} is not mangled (at least not yet) so needs to contain characters that are valid in a filename. Because the actual file will be stored with mangled names it is @@ -68,7 +76,7 @@ exists (or no mangledness if creating a new storr). \section{Corrupt keys}{ -Some file synchronisation utilities like dropbox can create file +Some file synchronisation utilities like Dropbox can create file that confuse an rds storr (e.g., \code{"myobject (Someone's conflicted copy)"}. If \code{mangle_key} is \code{FALSE} these cannot be detected but at @@ -83,9 +91,8 @@ the files that are causing the problem. Alternatively, you can try (assuming a storr object \code{st}) running -\preformatted{ -st$driver$purge_corrupt_keys() -} +\if{html}{\out{
}}\preformatted{st$driver$purge_corrupt_keys() +}\if{html}{\out{
}} which will delete corrupted keys with no confirmation. The messages that are printed to screen will be printed by default at diff --git a/man/test_driver.Rd b/man/test_driver.Rd index 995b55c..8e2a413 100644 --- a/man/test_driver.Rd +++ b/man/test_driver.Rd @@ -8,8 +8,8 @@ test_driver(create) } \arguments{ \item{create}{A function with one arguments that when run with -\code{NULL} as the argument will create a new driver instance. -It will also be called with a driver (of the same type) as an +\code{NULL} as the argument will create a new driver instance. It +will also be called with a driver (of the same type) as an argument - in this case, you must create a new driver object pointing at the same underlying storage (see the examples). Depending on your storage model, temporary directories, @@ -32,9 +32,9 @@ The test suite is included in the package as The procedure for each test block is: \enumerate{ -\item{Create a new driver by running \code{dr <- create()}.} -\item{Run a number of tests.} -\item{Destroy the driver by running \code{dr$destroy()}.} +\item Create a new driver by running \code{dr <- create()} +\item Run a number of tests. +\item Destroy the driver by running \code{dr$destroy()} } So before running this test suite, make sure this will not harm diff --git a/update_web.sh b/update_web.sh deleted file mode 100755 index ce504c3..0000000 --- a/update_web.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -e - -DOCS_DIR=docs -VERSION=$(git rev-parse --short HEAD) -REMOTE_URL=$(git config --get remote.origin.url) - -mkdir -p ${DOCS_DIR} -rm -rf ${DOCS_DIR}/.git -git init ${DOCS_DIR} -git -C ${DOCS_DIR} checkout --orphan gh-pages -git -C ${DOCS_DIR} add . -git -C ${DOCS_DIR} commit --no-verify -m "Update docs for version ${VERSION}" -git -C ${DOCS_DIR} remote add origin -m "gh-pages" ${REMOTE_URL} -git -C ${DOCS_DIR} push --force -u origin gh-pages diff --git a/vignettes/external.Rmd b/vignettes/external.Rmd index ff0b065..4a14747 100644 --- a/vignettes/external.Rmd +++ b/vignettes/external.Rmd @@ -25,7 +25,7 @@ key refers to a set of arguments to a long running function we get something like memoisation (see the bottom of this file). As an example, this vignette will download some DESCRIPTION files -from github, using the name of the repository as the key. +from GitHub, using the name of the repository as the key. The first step is writing a hook function; this is a function with arguments `(key, namespace)` that returns an R object. For @@ -78,7 +78,7 @@ As with other storr creation functions, you can set the default namespace using the `default_namespace` argument. The returned object is exactly the same as a usual storr except -that the `get` method has changed (this is done by inheritence). +that the `get` method has changed (this is done by inheritance). The `get` method only behaves differently when the object is not present in the storr, in which case it will try to fetch the object and insert it into the storr. diff --git a/vignettes/storr.Rmd b/vignettes/storr.Rmd index c5e7fe3..35e53d7 100644 --- a/vignettes/storr.Rmd +++ b/vignettes/storr.Rmd @@ -58,7 +58,7 @@ What is in the `storr`? st$list() ``` -Or, much faster, test for existance of a particular key: +Or, much faster, test for existence of a particular key: ``` {r } st$exists("mykey") @@ -219,19 +219,19 @@ st2$list() store the data. This is not the fastest interface but allows for interprocess key/value stores where a relational database is supported. All databases supported by DBI are supported (so at - least SQLite, MySQL and Postgres). + least `SQLite`, `MySQL` and `Postgres`). * Redis (`driver_redis`) - uses [`redux`](https://github.com/richfitz/redux) to store the data in a Redis (`http://redis.io`) database. About the same speed as rds (faster write, slower read at present), but can allow multiple R processes to share the same set of objects. -* rlite (`driver_rlite`) - stores data in an - [rlite](https://github.com/seppo0010/rlite) database using +* `rlite` (`driver_rlite`) - stores data in an + [`rlite`](https://github.com/seppo0010/rlite) database using [`rrlite`](https://github.com/ropensci/rrlite). This is quite quick, but is stalled for general release because `rrlite` does not support windows. * The in-development package - [thor](https://github.com/richfitz/thor) provides an alternative + [`thor`](https://github.com/richfitz/thor) provides an alternative on-disk storr that can handle multiple processes on a single machine. @@ -276,7 +276,7 @@ if present stores it and save it into the caching environment Because looking up data in the environment is likely to be orders -of magnitide faster than reading from disks or databases, this +of magnitude faster than reading from disks or databases, this means that commonly accessed data will be accessed at a similar speed to native R objects, while still immediately reflecting changes to the content (because that would mean the hash changes) @@ -286,7 +286,7 @@ To demonstrate: st <- storr::storr(driver = storr::driver_rds(tempfile("storr_"))) ``` -This is the caching environent; currently empty +This is the caching environment; currently empty ``` {r } ls(st$envir) ``` @@ -347,7 +347,7 @@ If a key is not in the database, storr will return a `KeyError` reasonable thing to do). If you _did_ want to return `NULL` when a key is requested but not -present, use tryCatch in this way: +present, use `tryCatch` in this way: ``` {r } tryCatch(st$get("no_such_key"), KeyError = function(e) NULL) diff --git a/vignettes_src/drivers.R b/vignettes_src/drivers.R deleted file mode 100644 index 4b948bc..0000000 --- a/vignettes_src/drivers.R +++ /dev/null @@ -1,489 +0,0 @@ -## --- -## title: "storr drivers" -## author: "Rich FitzJohn" -## date: "`r Sys.Date()`" -## output: rmarkdown::html_vignette -## vignette: > -## %\VignetteIndexEntry{storr drivers} -## %\VignetteEngine{knitr::rmarkdown} -## \usepackage[utf8]{inputenc} -## --- - -## Requirements for storr drivers. - -## The idea here is that you implement a handful of methods and the -## package will construct a common interface around them. There are -## built-in tests in the package to ensure that the driver behaves -## correctly, and infrastructure to help with running those tests. - -## To demonstrate we'll write a wrapper around RSQLite to store data -## (NOTE: this has since been rolled into the package as an actual -## driver `driver_dbi` but I'm leaving it here for posterity). - -## # How it works and what we need - -## First, consider the `get` method in storr. With a driver `dr`, -## storr retrieves values by running (approximately): - -## ```r -## dr$exists_hash(key, namespace) -## hash <- dr$get_hash(key, namespace) -## dr$exists_object(hash) -## dr$get_object(hash) -## ``` - -## which: -## -## 1. checks that a key exists (keys being defined by a combination of -## key and namespace) -## 2. retrieves the object hash stored against that key -## 3. checks that the hash is actually present in the database -## 4. retrieves the object stored against the hash -## -## hashes are stored as strings, while objects are *serialized R -## objects*, usually stored in binary. The driver is responsible for -## serialization/deserialization as that will depend on the properties -## of the driver. -## -## storr will take care of throwing appropriate errors if the object -## is not found (which requires the calls to `exists_hash` and -## `exists_object`). - -## `set` works in a similar way: -## -## ```r -## hash <- storr:::hash_object(hash) -## if (!dr$exists_object(hash)) { -## dr$set_object(hash, value) -## } -## dr$set_hash(key, namespace, hash) -## ``` -## -## The important part here is that storr will avoid setting the object -## if it can be avoided (i.e., if the hash is present in the database -## then the object has already been stored -- because saving the -## actual data is likely to be the slowest part it's worth avoiding). -## -## 1. if the hash is not present, save the (serialized) object against -## the hash. -## 2. store the hash against the key and namespace. - -## The full list of functions needed follows the next section. - -## ## A digression: key/value stores and SQL - -## SQL databases are probably not going to be a great place to store -## key/value data (especially very large objects) and this section is -## not meant to be normative. Instead, this is one possible route -## that could be taken. Recent version of postgresql include -## interfaces that support first class key/value (`hstore`) which -## would be prefereable to this. - -## Start with a SQLite connection (similar things can be done with -## other DBI drivers but at present this uses one SQLite-only function -## below): -con <- DBI::dbConnect(RSQLite::SQLite(), ":memory:") - -## This will create a table into which we can put key/value pairs. -table <- "mydata" -sql <- c(sprintf("CREATE TABLE IF NOT EXISTS %s", table), - "(name string PRIMARY KEY,", - "value blob)") -DBI::dbExecute(con, paste(sql, collapse = " ")) - -## Then take an object, serialize it, and stuff it into the blob, then -## insert that into the table. This is the part that varies between -## versions. - -## For both versions, we'll store the *value* of `mtcars` with the -## *name* `"mtcars"`: -value <- mtcars -name <- "mtcars" -sql <- sprintf("INSERT into %s (name, value) values (:name, :value)", table) - -dat <- list(name = name, value = list(serialize(value, NULL))) -DBI::dbExecute(con, sql, dat) - -## The pattern here is to use `dbExecute` to create and execute the -## query, injecting the raw byte sequence of the serialized object -## into the `value` column. This is currently supported only for -## RSQLite in the current release, but will probably be supported by -## other DBI-compatible packages over time. - -## We can retrieve the data by name: -sql <- sprintf('SELECT value FROM %s WHERE name == "%s"', table, name) -x <- unserialize(DBI::dbGetQuery(con, sql)[[1]][[1]]) -identical(x, value) - -## ## Implementing the storr driver - -## For consistency with other `storr` functions, even though the -## driver is an R6 object, we construct it using a plain R function -## (not the `$new()` method of an R6 generator. This is because the -## driver *could* be implemented as a function that generates a list -## of closures, a reference class, or some other approach. - -## There are quite a few required functions to implement. Below, the -## value in parentheses after the function signature is the expected -## return type. - -## * `type()` (returns character): Takes no argument and returns a -## string with the "type" of the driver. This can be any -## identifier. -## * `destroy()` (returns NULL): Takes no argument and destroys all -## data associated with the driver. Best to also leave the driver -## in a state where it can't be used, though this is not enforced. - -## * `set_hash(key, namespace, hash)` (returns NULL): Given strings -## for `key`, `namespace` and `hash`, set the `hash` against the -## `key`/`namespace` pair. -## * `get_hash(key, namespace)` (returns character): Given strings for -## `key`, and `namespace`, return the hash of the object. - -## * `set_object(hash, value)` (returns NULL): Given a string `hash` -## and an arbitrary object `value`, store the object against the -## hash. Serialization will likely be needed here (e.g., -## `serialize(value, NULL)`). -## * `get_object(hash)` (returns object): Given a string `hash` return -## the R object stored against it. Deserialization will likely be -## needed here (e.g., `unserialize(dat)`) - -## * `exists_hash(key, namespace)` (returns logical): Given vectors of -## strings for `key` and `namespace` return a vector with `TRUE` if -## there is a hash stored against the key/namespace pair, `FALSE` -## otherwise. -## * `exists_object(hash)` (returns logical): Given a vector of -## strings for `hash`, return a vector with `TRUE` if there is an -## object stored against the hash. - -## * `del_hash(key, namespace)` (returns logical): Given strings for -## `key` and `namespace`, delete this key if it exists. Return -## `TRUE` if the key existed, `FALSE` otherwise. -## * `del_object(hash)` (returns logical): Given a string for `hash` -## the object if it exists. Return `TRUE` if the hash existed, -## `FALSE` otherwise. - -## * `list_hashes()` (returns character vector): Return a character -## vector of all known hashes. -## -## * `list_namespaces()` (returns character vector): Return a -## character vector of all known namespaces. -## * `list_keys(namespace)` (returns character vector): Given a string -## `namespace`, return a character vector of all known keys within -## the namespace. - -## The arguments to the constructor will be -## -## * `con`: A `SQLiteConnection` object -## * `tbl_data`: Name of the table to store the data (hash/object pairs) in. -## * `tbl_keys`: Name of the table to store the keys -## (key/namespace/hash triplets) in. - -## Taking a `SQLiteConnection` object, rather than a path, saves -## duplicating the SQLite constructor function. - -## In addition, some care is required with the `hash_algorithm` -## argument -- see below. - -## The SQL queries are a bit ugly but hopefully straightforward enough -## to follow. - -driver_sqlite <- function(path, tbl_data = "storr_data", - tbl_keys = "storr_keys") { - R6_driver_sqlite$new(path, tbl_data, tbl_keys) -} - -## The R6 class definition that implements the functions above, with a -## little commentry throughout. -R6_driver_sqlite <- R6::R6Class( - "driver_sqlite", - public = list( - ## Public data members - con = NULL, - tbl_data = NULL, - tbl_keys = NULL, - - ## There is support for selecting different hash algorithms, - ## However, this requires that the driver stores the used alorithm - ## and errors if the algorithm changes, which is fiddly to set up. - ## So this tells storr that this driver does not support setting - ## the hash algorithm. - traits = list(hash_algorithm = FALSE), - - ## On initialisation we'll create the two tables but only if they - ## do not exist. We can enforce the constraint that hash must be - ## unique within tbl_data and key/namespace pairs must be unique - ## within tbl_keys. - initialize = function(con, tbl_data, tbl_keys) { - self$con <- con - self$tbl_data <- tbl_data - self$tbl_keys <- tbl_keys - - sql <- c(sprintf("CREATE TABLE if NOT EXISTS %s", tbl_data), - "(hash string PRIMARY KEY NOT NULL,", - "value blob NOT NULL)") - DBI::dbExecute(self$con, paste(sql, collapse = " ")) - - sql <- c(sprintf("CREATE TABLE IF NOT EXISTS %s", tbl_keys), - "(namespace string NOT NULL,", - "key string NOT NULL,", - "hash string NOT NULL,", - "PRIMARY KEY (namespace, key))") - DBI::dbExecute(self$con, paste(sql, collapse = " ")) - }, - - ## This is purely for identification later. - type = function() { - "DBI/sqlite" - }, - - ## Total destruction of the driver; delete all data stored in both - ## tables, then delete our database connection to render the - ## driver useless. - destroy = function() { - DBI::dbRemoveTable(self$con, self$tbl_data) - DBI::dbRemoveTable(self$con, self$tbl_keys) - self$con <- NULL - }, - - ## Return the hash value given a key/namespace pair - get_hash = function(key, namespace) { - sql <- sprintf( - 'SELECT hash FROM "%s" WHERE namespace = "%s" AND key = "%s"', - self$tbl_keys, namespace, key) - DBI::dbGetQuery(self$con, sql)[[1]] - }, - - ## Set the key/namespace pair to a hash - set_hash = function(key, namespace, hash) { - sql <- c(sprintf("INSERT OR REPLACE INTO %s", self$tbl_keys), - sprintf('(namespace, key, hash) VALUES ("%s", "%s", "%s")', - namespace, key, hash)) - DBI::dbExecute(self$con, paste(sql, collapse = " ")) - }, - - ## Return a (deserialized) R object, given a hash - get_object = function(hash) { - sql <- c(sprintf("SELECT value FROM %s", self$tbl_data), - sprintf('WHERE hash = "%s"', hash)) - value <- DBI::dbGetQuery(self$con, paste(sql, collapse = " "))[[1]] - unserialize(value[[1]]) - }, - - ## Set a (serialized) R object against a hash. This would be - ## considerably simpler (but probably slower and less accurate) if we - ## serialized to string with: - ## rawToChar(serialize(value, NULL, TRUE)) - set_object = function(hash, value) { - sql <- paste(sprintf("INSERT OR REPLACE INTO %s", self$tbl_data), - "(hash, value) VALUES (:hash, :value)") - dat <- list(hash = hash, value = list(serialize(value, NULL))) - DBI::dbExecute(self$con, sql, dat) - }, - - ## Check if a key/namespace pair exists. This is somewhat more - ## complicated than the other methods because storr assumes that - ## this can be done for vector arguments key and namespace. storr - ## provides a helper function 'join_key_namespace' for predictably - ## recycling keys and namespaces and then we consider - ## possibilities of 0, 1 or more items to lookup. The "more" case - ## can be done more efficiently with a single SQL statement, but - ## instead here we recurse. - exists_hash = function(key, namespace) { - nk <- storr::join_key_namespace(key, namespace) - if (nk$n == 0L) { - logical(0) - } else if (nk$n == 1L) { - sql <- sprintf('SELECT 1 FROM %s WHERE namespace = "%s" AND key = "%s"', - self$tbl_keys, namespace, key) - nrow(DBI::dbGetQuery(self$con, sql)) > 0L - } else { - vapply(seq_len(nk$n), function(i) - self$exists_hash(nk$key[[i]], nk$namespace[[i]]), - logical(1), USE.NAMES = FALSE) - } - }, - - ## Check if a hash exists. As for 'exists_hash' this must deal - ## with vectorised input but it's a little simpler. - exists_object = function(hash) { - if (length(hash) == 0L) { - logical(0) - } else if (length(hash) == 1L) { - sql <- sprintf('SELECT 1 FROM %s WHERE hash = "%s"', - self$tbl_data, hash) - nrow(DBI::dbGetQuery(self$con, sql)) > 0L - } else { - vapply(hash, self$exists_object, logical(1), USE.NAMES = FALSE) - } - }, - - ## Delete a key. Because of the requirement to return TRUE/FALSE on - ## successful/unsuccessful key deletion this includes an exists_hash() - ## step first. As with 'exists_hash' this needs to be vectorised. - del_hash = function(key, namespace) { - nk <- storr::join_key_namespace(key, namespace) - if (nk$n == 0L) { - logical(0) - } else if (nk$n == 1L) { - if (self$exists_hash(key, namespace)) { - sql <- sprintf('DELETE FROM %s WHERE namespace = "%s" AND key = "%s"', - self$tbl_keys, namespace, key) - DBI::dbExecute(self$con, sql) - TRUE - } else { - FALSE - } - } else { - vapply(seq_len(nk$n), function(i) - self$del_hash(nk$key[[i]], nk$namespace[[i]]), - logical(1), USE.NAMES = FALSE) - } - }, - - ## Delete a hash - del_object = function(hash) { - if (length(hash) == 0L) { - logical(0) - } else if (length(hash) == 1L) { - if (self$exists_object(hash)) { - sql <- sprintf( - 'DELETE FROM %s WHERE hash = "%s"', self$tbl_data, hash) - DBI::dbExecute(self$con, sql) - TRUE - } else { - FALSE - } - } else { - vapply(hash, self$del_object, logical(1), USE.NAMES = FALSE) - } - }, - - ## List hashes, namespaces and keys. Because the SQLite driver seems to - ## return numeric(0) if the result set is empty, we need as.character here. - list_hashes = function() { - sql <- sprintf("SELECT hash FROM %s", self$tbl_data) - as.character(DBI::dbGetQuery(self$con, sql)[[1]]) - }, - - list_namespaces = function() { - sql <- sprintf("SELECT DISTINCT namespace FROM %s", self$tbl_keys) - as.character(DBI::dbGetQuery(self$con, sql)[[1]]) - }, - - list_keys = function(namespace) { - sql <- sprintf('SELECT key FROM %s WHERE namespace = "%s"', - self$tbl_keys, namespace) - as.character(DBI::dbGetQuery(self$con, sql)[[1]]) - } - )) - -## Next, let's give the driver a little workout. -con <- DBI::dbConnect(RSQLite::SQLite(), ":memory:") -dr <- driver_sqlite(con) - -## Start with the hash part of the database. At first we have no -## hashes in the database: -dr$list_hashes() - -## so `exists_object` returns `FALSE`: -hash <- digest::digest(mtcars) -dr$exists_object(hash) - -## We can set an object against a hash: -dr$set_object(hash, mtcars) - -## and then `exists_object` will return `TRUE` -dr$exists_object(hash) - -## and we can retrieve the object: -head(dr$get_object(hash)) - -## Our set of hashes: -dr$list_hashes() - -## Delete the hash: -dr$del_object(hash) - -## And it's gone: -dr$list_hashes() -dr$exists_object(hash) - -## Set up a key -key <- "aaa" -namespace <- "ns" -dr$set_hash(key, namespace, hash) - -## Which now exists: -dr$exists_hash(key, namespace) - -## And can be listed: -dr$list_keys(namespace) - -## and the hash against the key returned: -dr$get_hash(key, namespace) - -dr$del_hash(key, namespace) -dr$exists_hash(key, namespace) -dr$list_keys(namespace) - -## OK, so this *seems* to be working. But how do we test if it is -## actually working? `storr` provides an automatic testing facility -## based on `testthat`. - -## To do this, we provide a function that takes one argument `dr`; if -## this is `NULL` then we must create a new empty driver, if -## non-`NULL` we must create a driver pointing at the same storage as -## an existing driver `dr`: -create_sqlite <- function(dr = NULL) { - if (is.null(dr)) { - con <- DBI::dbConnect(RSQLite::SQLite(), ":memory:") - } else { - con <- dr$con - } - driver_sqlite(con) -} - -## (because this function will repeatedly create new databases, -## `:memory:` is a good path here)! - -## Pass this in to `storr::test_driver` -storr::test_driver(create_sqlite) - -## Now that the driver works, we can write the wrapper function: -storr_sqlite <- function(con, - tbl_data = "storr_data", tbl_keys = "storr_keys", - default_namespace = "objects") { - storr::storr(driver_sqlite(con, tbl_data, tbl_keys), - default_namespace) -} - -## and construct a `storr` with it: -st_sql <- storr_sqlite(DBI::dbConnect(RSQLite::SQLite(), ":memory:")) - -## Nothing in the storr: -st_sql$list() - -## Set some data: -st_sql$set("foo", runif(10)) - -## Retrieve it: -st_sql$get("foo") - -## Delete it: -st_sql$del("foo") -st_sql$list() - -## Underlying data is still kicking around: -st_sql$list_hashes() -st_sql$get_value(st_sql$list_hashes()) - -## But we can garbage collect: -st_sql$gc() -st_sql$list_hashes() - -st_sql$destroy() - -## This is not really SQL's strong suit. But if key/value storage is -## a small part of an application that already uses SQLite for storage -## then this approach could be a sensible move. diff --git a/vignettes_src/external.R b/vignettes_src/external.R deleted file mode 100644 index 0be4e9f..0000000 --- a/vignettes_src/external.R +++ /dev/null @@ -1,212 +0,0 @@ -## --- -## title: "external" -## author: "Rich FitzJohn" -## date: "`r Sys.Date()`" -## output: rmarkdown::html_vignette -## vignette: > -## %\VignetteIndexEntry{external} -## %\VignetteEngine{knitr::rmarkdown} -## \usepackage[utf8]{inputenc} -## --- - -## Using external storrs. - -## Often it is useful to retrieve data from an external resource -## (especially websites). The way this works is: -## -## 1. We do a key lookup on the storr; if that succeeds (i.e. it maps -## to a hash) continue as normal.. -## -## 2. If the lookup fails, pass the key (and namespace) to a "hook" -## function that generates an R object (in any way). - -## This is in some ways a variant on the memoisation pattern; if the -## key refers to a set of arguments to a long running function we get -## something like memoisation (see the bottom of this file). - -## As an example, this vignette will download some DESCRIPTION files -## from github, using the name of the repository as the key. - -## The first step is writing a hook function; this is a function with -## arguments `(key, namespace)` that returns an R object. For -## packages stored in the root directory of a repository we can build -## URLs of the form -## -## ``` -## https://raw.githubusercontent.com///master/DESCRIPTION -## ``` -## -## So if the key is a username/repo pair and we ignore namespace we -## can write a function: -fetch_hook_gh_description <- function(key, namespace) { - if (!isTRUE(unname(capabilities("libcurl")))) { - stop("This vignette requires libcurl support in R to run") - } - fmt <- "https://raw.githubusercontent.com/%s/master/DESCRIPTION" - path <- tempfile("gh_description_") - on.exit(file.remove(path)) - code <- download.file(sprintf(fmt, key), path, mode = "wb") - if (code != 0L) { - stop("Error downloading file") - } - as.list(read.dcf(path)[1, ]) -} - -## This function downloads the requested DESCRIPTION file into a -## temporary file (which it promises to delete later using `on.exit`), -## checks that the download was successful, then reads in the -## downloaded file and converts it into a list. - -## The `httr` and `curl` packages make this a little easier to do with -## authorisation so that this would work for private repositories by -## using a token. - -## With this in place, we can build a storr: -st <- storr::storr_external(storr::driver_environment(), - fetch_hook_gh_description) - -## The first argument here is a storr *driver* (i.e., a `driver_` -## function). If you have a storr that you want to use, pass it as -## `st$driver` to extract the underlying driver (and share storage -## with your existing storr). -## -## As with other storr creation functions, you can set the default -## namespace using the `default_namespace` argument. -## -## The returned object is exactly the same as a usual storr except -## that the `get` method has changed (this is done by inheritence). -## The `get` method only behaves differently when the object is not -## present in the storr, in which case it will try to fetch the object -## and insert it into the storr. - -## At first there is nothing in here: -st$list() - -## But we can still `get` things from the storr: -d <- st$get("richfitz/storr") - -## Once a key has been fetched, it will be retrieved locally: -identical(st$get("richfitz/storr"), d) - -## And it will be present within the storr, as shown by `list`: -st$list() - -## If an external resource cannot be located, storr will throw an -## error of class `KeyErrorExternal`: -tryCatch(st$get("richfitz/no_such_repo"), - KeyErrorExternal = function(e) - message(sprintf("** Repository %s not found", e$key))) - -## This would happen for all errors, including lack of internet -## connectivity, corrupt file downloads, etc. The original error will -## be returned as the `$e` element of the error if you need to -## distinguish between types of failure. The `KeyErrorExternal` is -## also a `KeyError` class, so code that catches `KeyErrors` will -## still work as expected. - -## For more details on storr exception handling, see the `storr` -## vignette (`vignette("storr", package = "storr")`) - -## Note that if you want to persist the storage of the descriptions -## you would need to mangle the key: -st_rds <- st$export(storr::storr_rds(tempfile(), mangle_key = TRUE)) -st_rds$list() -st_rds$get("richfitz/storr")$Version - -## The `st_rds` storr does not include the fetch hook; it is a plain storr. - -###+ echo = FALSE -st_rds$destroy() - -## # Memoisation - -## The external storr can support a form of memoisation, though it -## might be simpler to implement this directly (see below). - -## Suppose you have some expensive function `f(a, b)` -f <- function(a, b) { - message(sprintf("Computing f(%.3f, %.3f)", a, b)) - ## ...expensive computation here... - list(a, b) -} - -## and a set of parameters to run the function over, with each -## parameter set (row) associated with an id: -pars <- data.frame(id = as.character(1:10), a = runif(10), b = runif(10), - stringsAsFactors = FALSE) - -## The `hook` here simply looks the parameters up and arranges to run them: -hook <- function(key, namespace) { - p <- pars[match(key, pars$id), -1] - f(p$a, p$b) -} - -st <- storr::storr_external(storr::driver_environment(), hook) - -## The first time the result is retrieved the message will be printed -## (the function is evaluated) -x <- st$get("1") - -## The second time, it will not be as the result is retrieved from the -## storr: -identical(st$get("1"), x) - -## This idea can be generalised by storing the *parameters* and the -## *functions* in the storr so that we lose the dependency on the -## global variables: -st <- storr::storr_environment() -st$set("experiment1", pars, namespace = "parameters") -st$set("experiment1", f, namespace = "functions") - -hook2 <- function(key, namespace) { - f <- st$get(namespace, namespace = "functions") - pars <- st$get(namespace, namespace = "parameters") - p <- pars[match(key, pars$id), -1] - f(p$a, p$b) -} - -st_use <- storr::storr_external(st$driver, hook2) - -x1 <- st_use$get("1", "experiment1") -x2 <- st_use$get("1", "experiment1") - -## Memoisation in the style of the `memoise` package is possible to -## implement, but is not provided in the package. Implementation is -## straightforward and will work with any driver: -memoise <- function(f, driver = storr::driver_environment()) { - force(f) - st <- storr::storr(driver) - function(...) { - ## NOTE: also digesting the inputs as a key here (in addition to - ## storr's usual digesting of values) - key <- digest::digest(list(...)) - tryCatch( - st$get(key), - KeyError = function(e) { - ans <- f(...) - st$set(key, ans) - ans - }) - } -} - -## Here's a function that will print when it is evaluated -f <- function(x) { - message("computing...") - x * 2 -} - -## Create the memoised function -g <- memoise(f) - -## The first time an argument is seen, `f()` will be run, printing a message -g(1) - -## Subsequent times will be looked up from the storr: -g(1) - -## Storr takes about twice as long as memoise (memoise does a direct -## key->value mapping rather than going through hashed values because -## it is the only thing that ever touches its cache). However, the -## overhead is approximately half of one call to `message()` so it's -## not that bad. diff --git a/vignettes_src/storr.R b/vignettes_src/storr.R deleted file mode 100644 index 67a03d1..0000000 --- a/vignettes_src/storr.R +++ /dev/null @@ -1,307 +0,0 @@ -## --- -## title: "storr" -## author: "Rich FitzJohn" -## date: "`r Sys.Date()`" -## output: rmarkdown::html_vignette -## vignette: > -## %\VignetteIndexEntry{storr} -## %\VignetteEngine{knitr::rmarkdown} -## \usepackage[utf8]{inputenc} -## --- - -library(storr) - -## `storr` provides very simple key/value stores for R. They attempt -## to provide the most basic set of key/value lookup functionality -## that is completely consistent across a range of different -## underlying storage drivers (in memory storage, filesystem and -## proper databases). All the storage is _content addressable_, so -## keys map onto hashes and hashes map onto data. - -## The `rds` driver stores contents at some path by saving out to rds -## files. Here I'm using a temporary directory for the path; the -## driver will create a number of subdirectories here. -path <- tempfile("storr_") -st <- storr::storr_rds(path) - -## Alternatively you can create the driver explicitly: -##+ eval=FALSE -dr <- storr::driver_rds(path) - -## With this driver object we can create the `storr` object which is -## what we actually interact with: -##+ eval=FALSE -st <- storr::storr(dr) - -## ## Key-value store - -## The main way of interacting with a `storr` object is -## `get`/`set`/`del` for getting, setting and deleting data stored at -## some key. To store data: -st$set("mykey", mtcars) - -## To get the data back -head(st$get("mykey")) - -## What is in the `storr`? -st$list() - -## Or, much faster, test for existance of a particular key: -st$exists("mykey") - -st$exists("another_key") - -## To delete a key: -st$del("mykey") - -## It's gone! -st$list() - -## though the actual data is still stored in the database: -h <- st$list_hashes() -h - -## The hash of an object is computed using the `digest` package, and -## can be done using the `hash_object` method of the storr. -st$hash_object(mtcars) - -## An object can be retrieved directly given its hash: -head(st$get_value(h)) - -## similarly, we can test to see if an object is present in the -## database using its hash: -st$exists_object(h) - -## though now that there are no keys pointing at the data it is -## subject to garbage collection: -del <- st$gc() -del - -st$list_hashes() - -## ## Namespaces - -## At some point having everything stored in a great big bucket may -## become too unstructured. To help with this storr implements a very -## simple "namespace" system that may help provide some structure. It -## is a single layer of hierarchy above keys; so every key belongs to -## a namespace. The default namespace is "objects" but this can be -## configured when the storr is created. -st$default_namespace - -## The `list_namespaces()` method lists all known namespaces -st$list_namespaces() - -## To create a new namespace, simply assign an object into it: -st$set("a", runif(5), namespace = "other_things") -st$list_namespaces() - -## The `list()` method lists the contents of a single namespace -st$list() -st$list("other_things") - -## To get an object, you must use the correct namespace: -##+ error = TRUE -st$get("a") -st$get("a", "other_things") - -## ## Bulk get/set - -## If you have many values to get or set, for some databases it will -## be much more efficient to get and set them in bulk; this is -## particularly the case with high-latency databases (e.g., anything -## over a network connection, especially an internet connection). To -## help with this, storr implements `mget` and `mset` methods that -## allow multiple values to retrieved or set. - -## The `mset` function allows multiple keys (and/or multiple -## namespaces) and multiple data elements. The data must have the -## same `length()` as the number of keys being set. -st$mset(c("a", "b", "c"), list(1, 2, 3)) -st$get("a") - -## The `mget` function fetches zero or more elements. -st$mget(c("a", "b", "c")) - -## `mget` *always* returns a list with the same number of elements as -## the number of keys -st$mget("a") -st$mget(character(0)) - -## With both `mset` and `mget`, both key and namespace can be vectors; -## if either non-scalar, they must have the same length so the logic -## is fairly predictable -st$mset("x", list("a", "b"), namespace = c("ns1", "ns2")) -st$mget("x", c("ns1", "ns2")) - -st$mget(c("a", "b", "x"), c("objects", "objects", "ns1")) - -## ## Import / export - -## Objects can be imported in and exported out of a `storr`: - -## Import from a list, environment or another `storr` -st$import(list(a = 1, b = 2)) -st$list() -st$get("a") - -## Export to an environment (or another `storr`) -e <- st$export(new.env(parent = emptyenv())) -ls(e) -e$a - -st_copy <- st$export(storr_environment()) -st_copy$list() -st$get("a") - -st2 <- storr::storr(driver = storr::driver_rds(tempfile("storr_"))) -st2$list() -st2$import(st) -st2$list() - -## ## Supported backends - -## * environments (`driver_environment`) - mostly for debugging and -## transient storage, but by far the fastest. -## * on disk with rds (`driver_rds`) - zero dependencies, quite fast, -## will suffer under high concurrency because there is no file -## locking. -## * DBI (`driver_dbi`) - uses (abuses?) a relational database to -## store the data. This is not the fastest interface but allows for -## interprocess key/value stores where a relational database is -## supported. All databases supported by DBI are supported (so at -## least SQLite, MySQL and Postgres). -## * Redis (`driver_redis`) - uses -## [`redux`](https://github.com/richfitz/redux) to store the data in a -## Redis (`http://redis.io`) database. About the same speed as rds -## (faster write, slower read at present), but can allow multiple R -## processes to share the same set of objects. -## * rlite (`driver_rlite`) - stores data in an -## [rlite](https://github.com/seppo0010/rlite) database using -## [`rrlite`](https://github.com/ropensci/rrlite). This is quite -## quick, but is stalled for general release because `rrlite` does not -## support windows. -## * The in-development package -## [thor](https://github.com/richfitz/thor) provides an alternative -## on-disk storr that can handle multiple processes on a single -## machine. - -## ## Implementation details - -## `storr` includes a few useful features that are common to all -## drivers. - -## ### Content addressable lookup - -## The only thing that is stored against a key is the hash of some -## object. Each driver does this a different way, but for the rds -## driver it stores small text files that list the hash in them. So: -dir(file.path(path, "keys", "objects")) -readLines(file.path(path, "keys", "objects", "a")) -st$get_hash("a") - -## Then there is one big pool of hash / value pairs: -st$list_hashes() - -## in the rds driver these are stored like so: -dir(file.path(path, "data")) - -## ### Environment-based caching - -## Every time data passes across a `get` or `set` method, `storr` -## stores the data in an environment within the `storr` object. -## Because we store the content against its hash, it's always in sync -## with what is saved to disk. That means that the lookup process -## goes like this: -## -## 1. Ask for a key, get returned the hash of the content -## 2. Check in the caching environment for that hash and return that -## if present -## 3. If not present, read content from disk/db/wherever the driver -## stores it and save it into the caching environment -## -## Because looking up data in the environment is likely to be orders -## of magnitide faster than reading from disks or databases, this -## means that commonly accessed data will be accessed at a similar -## speed to native R objects, while still immediately reflecting -## changes to the content (because that would mean the hash changes) - -## To demonstrate: -st <- storr::storr(driver = storr::driver_rds(tempfile("storr_"))) - -## This is the caching environent; currently empty -ls(st$envir) - -## Set some key to some data: -set.seed(2) -st$set("mykey", runif(100)) - -## The environment now includes an object with a *name* that is the -## same as the *hash* of its contents: -ls(st$envir) - -## Extract the object from the environment and hash it -st$hash_object(st$envir[[ls(st$envir)]]) - -## When we look up the value stored against key `mykey`, the first -## step is to check the key/hash map; this returns the key above (this -## step *does* involve reading from disk) -st$get_hash("mykey") - -## It then calls `$get_value` to extract the value associated with -## that hash - the first thing that function does is try to locate the -## hash in the environment, otherwise it reads the data from wherever -## the driver stores it. -st$get_value - -## The speed up is going to be fairly context dependent, but 5-10x -## seems pretty good in this case (some of the overhead is simply a -## longer code path as we call out to the driver). For big bits of -## data and slow network connections the difference will be much more -## pronounced. -hash <- st$get_hash("mykey") -if (requireNamespace("rbenchmark")) { - rbenchmark::benchmark(st$get_value(hash, use_cache = TRUE), - st$get_value(hash, use_cache = FALSE), - replications = 1000, order = NULL)[1:4] -} - -## ### Classed exceptions - -## storr uses R's exception handling system and errors inspired from -## Python to make it easy to program with `tryCatch`. - -## If a key is not in the database, storr will return a `KeyError` -## (not `NULL` because storing a `NULL` value is a perfectly -## reasonable thing to do). - -## If you _did_ want to return `NULL` when a key is requested but not -## present, use tryCatch in this way: -tryCatch(st$get("no_such_key"), - KeyError = function(e) NULL) - -## See `?tryCatch` for details. The idea is that key lookup errors -## will have the class `KeyError` so will be caught here and run the -## given function (the argument `e` is the actual error object). -## Other errors will not be caught and will still throw. - -## `HashErrors` will be rarer, but could happen (they might occur if -## your driver supports expiry of objects). We can simulate that by -## setting a hash and deleting it: -## -st$set("foo", letters) -ok <- st$driver$del_object(st$get_hash("foo")) -st$flush_cache() -tryCatch(st$get("foo"), - KeyError = function(e) NULL, - HashError = function(e) message("Data is deleted")) - -## Here the `HashError` is triggered. - -## `KeyError` objects include `key` and `namespace` elements, -## `HashError` objects include a `hash` element. They both inherit -## from `c("error", "condition")`. - -## Finally, when using an external storr (see ?driver_external) storr -## will throw a `KeyErrorExternal` if the `fetch_hook` function errors -## while trying to retrieve an external resource.