Initial Zarrs Implementation (#172)

Author: dblodgett-usgs

.Rbuildignore

@@ -16,3 +16,11 @@ r6-roxygen-convention.md
 cran-comments.md
 CONTRIBUTING.md
 ^CRAN-SUBMISSION$
+^dev\.md$
+^RUST-STYLE\.md$
+^\.vscode$
+^src/\.cargo$
+^src/rust/vendor$
+^src/rust/target$
+^src/Makevars$
+^src/Makevars\.win$

.gitattributes

@@ -1,3 +1,5 @@
 * text=auto
 *.zip binary
 tests/fixtures/**/* -diff
+configure eol=lf
+configure.win eol=crlf

.github/workflows/R-CMD-Check.yml

@@ -50,6 +50,7 @@ jobs:
       - uses: r-lib/actions/check-r-package@v2
         with:
           upload-snapshots: true
+          error-on: '"error"'
 
       - name: Test coverage
         run: covr::codecov()

.gitignore

@@ -23,3 +23,7 @@ cran-comments.md
 /.quarto/
 **/*.quarto_ipynb
 CRAN-SUBMISSION
+src/rust/vendor
+src/Makevars
+src/Makevars.win
+.vscode

CONTRIBUTING.md

@@ -9,16 +9,65 @@ devtools::install()
 devtools::load_all()
 ```
 
-## Testing
+## Build and Test Cycles
 
-Tests run single-threaded (`Config/testthat/parallel: false` in
-DESCRIPTION).
+pizzarr has two build tiers that affect how you develop and test locally.
+
+### CRAN tier (pure R)
+
+If you are working on R-side logic (indexers, stores, codecs, R6 classes)
+and do not need the zarrs backend, this is the faster cycle:
 
 ``` r
+devtools::load_all()
+devtools::test()
 devtools::check()
+```
+
+Tests run single-threaded (`Config/testthat/parallel: false` in
+DESCRIPTION). All zarrs-specific tests skip automatically when the Rust
+library is absent — they guard on `.pizzarr_env$zarrs_available`.
+
+To produce a CRAN-ready source tarball that strips `src/`, `configure`,
+and `SystemRequirements`:
+
+``` bash
+bash tools/cran-build.sh
+```
+
+The resulting `.tar.gz` passes `R CMD check` without a Rust toolchain.
+
+### r-universe tier (zarrs backend)
+
+The r-universe build compiles the [zarrs](https://github.com/zarrs/zarrs)
+Rust crate via [extendr](https://extendr.github.io/rextendr/). You need
+rustc >= 1.91 with the GNU target on Windows:
+
+``` bash
+rustup target add x86_64-pc-windows-gnu
+```
+
+After modifying any `#[extendr]` function in `src/rust/src/lib.rs`,
+regenerate the R wrappers:
+
+``` r
+rextendr::document()
+```
+
+This rebuilds the Rust library and regenerates `R/extendr-wrappers.R`.
+Never edit that file by hand. Then run the standard cycle:
+
+``` r
+devtools::load_all()
 devtools::test()
+devtools::check()
 ```
 
+`devtools::check()` will emit a NOTE about downloading Rust crates — that
+is expected and is not a problem for r-universe builds. See
+[RUST-STYLE.md](RUST-STYLE.md) for Rust conventions, module layout, and
+build pipeline details.
+
 ## Building Documentation
 
 ``` r

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: pizzarr
 Type: Package
 Title: Slice into 'Zarr' Arrays
-Version: 0.1.3
+Version: 0.2.0
 Authors@R: c(
     person(given = "David",
            family = "Blodgett",
@@ -29,8 +29,10 @@ Description: An implementation of chunked, compressed,
 Language: en-US
 License: MIT + file LICENSE
 BugReports: https://github.com/zarr-developers/pizzarr/issues
-URL: https://zarr.dev/pizzarr/
-Depends: R (>= 4.1.0)
+URL: https://zarr.dev/pizzarr/,
+    https://github.com/zarr-developers/pizzarr
+Depends: 
+    R (>= 4.2)
 Imports:
     jsonlite,
     stats,
@@ -42,6 +44,7 @@ Imports:
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE, r6 = TRUE)
 RoxygenNote: 7.3.3
+VignetteBuilder: knitr
 Suggests:
     testthat,
     knitr,
@@ -61,3 +64,5 @@ Suggests:
     withr
 Config/testthat/parallel: false
 Config/testthat/edition: 3
+Config/rextendr/version: 0.4.2
+SystemRequirements: Cargo (Rust's package manager), rustc >= 1.91

NAMESPACE

@@ -28,8 +28,10 @@ export(is_key_error)
 export(is_scalar)
 export(is_slice)
 export(obj_list)
+export(pizzarr_compiled_features)
 export(pizzarr_option_defaults)
 export(pizzarr_sample)
+export(pizzarr_upgrade)
 export(slice)
 export(zarr_create)
 export(zarr_create_array)
@@ -41,10 +43,19 @@ export(zarr_open_array)
 export(zarr_open_group)
 export(zarr_save_array)
 export(zarr_volcano)
+export(zarrs_close_store)
+export(zarrs_compiled_features)
+export(zarrs_get_subset)
+export(zarrs_node_exists)
+export(zarrs_open_array_metadata)
+export(zarrs_runtime_info)
+export(zarrs_set_codec_concurrent_target)
+export(zarrs_set_subset)
 export(zb_int)
 export(zb_slice)
 importFrom(R6,R6Class)
 importFrom(memoise,memoise)
 importFrom(memoise,timeout)
 importFrom(qs2,zstd_compress_raw)
 importFrom(qs2,zstd_decompress_raw)
+useDynLib(pizzarr, .registration = TRUE)

R/extendr-wrappers.R

@@ -0,0 +1,91 @@
+# Generated by extendr: Do not edit by hand
+
+# nolint start
+
+#
+# This file was created with the following call:
+#   .Call("wrap__make_pizzarr_wrappers", use_symbols = TRUE, package_name = "pizzarr")
+
+#' @usage NULL
+#' @useDynLib pizzarr, .registration = TRUE
+NULL
+
+#' Return compiled zarrs feature flags as a character vector.
+#'
+#' Called once at `.onLoad` to populate `.pizzarr_env$zarrs_available`.
+#' The feature list is determined at compile time via `cfg!` checks.
+#' @export
+zarrs_compiled_features <- function() .Call(wrap__zarrs_compiled_features)
+
+#' Check whether a node exists at the given path in a zarrs store.
+#'
+#' Open (or reuse) the store at `store_url`, then probe for V3 and V2
+#' metadata keys at `path`.
+#'
+#' @param store_url Filesystem path or URL to the store root.
+#' @param path Path within the store (e.g. `"group1/array1"` or `""` for root).
+#' @export
+zarrs_node_exists <- function(store_url, path) .Call(wrap__zarrs_node_exists, store_url, path)
+
+#' Close (remove) a cached store handle.
+#'
+#' Returns `TRUE` if the store was in the cache and was removed,
+#' `FALSE` if it was not cached.
+#'
+#' @param store_url Filesystem path or URL to the store root.
+#' @export
+zarrs_close_store <- function(store_url) .Call(wrap__zarrs_close_store, store_url)
+
+#' Open a zarrs array and return its metadata as an R list.
+#'
+#' Returns a named list with `shape`, `chunks`, `dtype`, `r_type`,
+#' `fill_value_json`, `zarr_format`, and `order`.
+#'
+#' @param store_url Filesystem path or URL to the store root.
+#' @param array_path Path to the array within the store.
+#' @export
+zarrs_open_array_metadata <- function(store_url, array_path) .Call(wrap__zarrs_open_array_metadata, store_url, array_path)
+
+#' Return runtime information about the zarrs backend.
+#'
+#' Returns a named list with `codec_concurrent_target`,
+#' `store_cache_entries`, and `compiled_features`.
+#' @export
+zarrs_runtime_info <- function() .Call(wrap__zarrs_runtime_info)
+
+#' Set the zarrs codec concurrent target.
+#'
+#' Controls the number of concurrent codec operations zarrs uses
+#' within a single array operation.
+#'
+#' @param n Positive integer.
+#' @export
+zarrs_set_codec_concurrent_target <- function(n) .Call(wrap__zarrs_set_codec_concurrent_target, n)
+
+#' Get a contiguous subset of an array.
+#'
+#' Returns a named list with `data` (numeric, integer, or logical vector)
+#' and `shape` (integer vector). Ranges are 0-based, exclusive stop.
+#'
+#' @param store_url Filesystem path or URL to the store root.
+#' @param array_path Path to the array within the store.
+#' @param ranges R list of length-2 integer vectors `c(start, stop)`.
+#' @param concurrent_target Optional codec concurrency override.
+#' @export
+zarrs_get_subset <- function(store_url, array_path, ranges, concurrent_target) .Call(wrap__zarrs_get_subset, store_url, array_path, ranges, concurrent_target)
+
+#' Set a contiguous subset of an array from R data.
+#'
+#' Returns `true` on success. Ranges are 0-based, exclusive stop.
+#' Data must be a flat vector in C-order (row-major).
+#'
+#' @param store_url Filesystem path or URL to the store root.
+#' @param array_path Path to the array within the store.
+#' @param ranges R list of length-2 integer vectors `c(start, stop)`.
+#' @param data R vector (numeric, integer, or logical).
+#' @param concurrent_target Optional codec concurrency override.
+#' @export
+zarrs_set_subset <- function(store_url, array_path, ranges, data, concurrent_target) .Call(wrap__zarrs_set_subset, store_url, array_path, ranges, data, concurrent_target)
+
+
+# nolint end

R/onload.R

@@ -1,4 +1,18 @@
 #' @keywords internal
 .onLoad <- function(libname = NULL, pkgname = NULL) {
-    init_options()
+  init_options()
+  .pizzarr_env$zarrs_available <- is_zarrs_available()
+}
+
+#' @keywords internal
+.onAttach <- function(libname, pkgname) {
+  if (!.pizzarr_env$zarrs_available &&
+      getOption("pizzarr.suggest_runiverse", TRUE)) {
+    packageStartupMessage(
+      "pizzarr: running without zarrs backend (pure R).\n",
+      "For parallel I/O and cloud storage, install from r-universe:\n",
+      '  install.packages("pizzarr", ',
+      'repos = "https://zarr-developers.r-universe.dev")'
+    )
+  }
 }

R/stores.R

@@ -142,6 +142,15 @@ Store <- R6::R6Class("Store",
       return(private$zmetadata)
     },
     #' @description
+    #' Return a store identifier for zarrs dispatch.
+    #'
+    #' Subclasses override this to return a filesystem path or URL.
+    #' Returns `NULL` by default (signals: use R-native path).
+    #' @return A character string or `NULL`.
+    get_store_identifier = function() {
+      NULL
+    },
+    #' @description
     #' Print a human-readable summary of the store.
     #' @param ... Ignored.
     #' @return `self` (invisibly).
@@ -259,6 +268,12 @@ DirectoryStore <- R6::R6Class("DirectoryStore",
     },
     #' @description
     #' Print a human-readable summary of the store.
+    #' @description
+    #' Return the absolute filesystem path for zarrs dispatch.
+    #' @return A character string.
+    get_store_identifier = function() {
+      normalizePath(self$root, mustWork = FALSE)
+    },
     #' @param ... Ignored.
     #' @return `self` (invisibly).
     print = function(...) {
@@ -655,6 +670,12 @@ HttpStore <- R6::R6Class("HttpStore",
     },
     #' @description
     #' Print a human-readable summary of the store.
+    #' @description
+    #' Return the store URL for zarrs dispatch.
+    #' @return A character string.
+    get_store_identifier = function() {
+      private$url
+    },
     #' @param ... Ignored.
     #' @return `self` (invisibly).
     print = function(...) {