diff --git a/NAMESPACE b/NAMESPACE index cd5cef8..8eb2220 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,5 +4,6 @@ export(convert_bytes_to_array) export(parse_npy_datatype) export(read_npy) export(read_npz) +importFrom(stats,setNames) importFrom(utils,unzip) useDynLib(grumpy, .registration = TRUE, .fixes = "C_") diff --git a/R/read_npy.R b/R/read_npy.R index 4c2efce..18537e7 100644 --- a/R/read_npy.R +++ b/R/read_npy.R @@ -1,15 +1,36 @@ -#' Read a .npy file +#' Read a `.npy` file #' -#' @param file Path to the .npy file +#' @param file Path to the `.npy` file #' @param ... Ignored. Reserved for future use. #' -#' @returns An array containing the data from the .npy file +#' @returns An array containing the data from the `.npy` file #' #' @export #' #' @examples +#' # Array of integers. NumPy "i8") #' parse_npy_datatype("|b1") +#' # A structured datatype where each element has 3 components, all integers, +#' # named "r", "g" and "b". #' parse_npy_datatype(list(c("r", " -#' convert_bytes_to_array("int", shape = c(2L, 3L), size = 4L, endian = "little") +#' convert_bytes_to_array( +#' "int", +#' shape = c(2L, 3L), +#' size = 4L, +#' endian = "little" +#' ) #' y #' dim(y) #' is.array(y) @@ -212,7 +246,7 @@ convert_bytes_to_array <- function(bytes, what, shape, size, endian) { by = record_size, length.out = n_records ) - idx <- rep(starts, each = size[[i]]) + seq_len(size[[i]]) - 1 + idx <- rep(starts, each = size[[i]]) + seq_len(size[[i]]) - 1L res_fields[[i]] <- convert_bytes_to_array( bytes[idx], what = what[[i]], diff --git a/R/read_npz.R b/R/read_npz.R index ecf2b19..bd287b9 100644 --- a/R/read_npz.R +++ b/R/read_npz.R @@ -1,10 +1,11 @@ -#' Read a .npz file +#' Read a `.npz` file #' -#' @param file Path to the .npz file +#' @param file Path to the `.npz` file #' -#' @return A list of arrays containing the data from the .npz file +#' @return A named list of arrays containing the data from the `.npz` file #' #' @importFrom utils unzip +#' @importFrom stats setNames #' #' @export #' @@ -24,5 +25,6 @@ read_npz <- function(file) { con <- unz(file, name, "rb") on.exit(close(con)) read_npy(con) - }) + }) |> + setNames(gsub("\\.npy$", "", files$Name)) } diff --git a/README.md b/README.md index d71740a..1e75bc6 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,17 @@ into R. It supports a wide range of data types and array shapes. As a file format generated by a Python package, `.npy` files are prime candidates for using the `{reticulate}` package to read them into R. -However, this comes with downsides in terms of performance, flexibility, -and robustness of the R package infrastructure. `{grumpy}`, on the other -hand, is a pure R package with no dependency and is performant, -flexible. Overall, it is designed to be used deep in the dependency -graph of other packages. +However, using a Python package in R comes with downsides in terms of: - +performance, since the data needs to be copied across languages / +sessions - flexibility, since reticulate makes opinionated choices to +work out of the box for most use cases - robustness, since packages +depending on reticulate need to risk breaking changes in the python +packages they now depend on, or need to provide an environment via +`{basilisk}`. + +`{grumpy}`, on the other hand, is a pure R package with no dependency +and is performant, flexible. Overall, it is designed to be used deep in +the dependency graph of other packages. For more details on the motivation and design principles underpinning `{grumpy}`, see the dedicated vignette: diff --git a/README.qmd b/README.qmd index a2292cf..e7b9de3 100644 --- a/README.qmd +++ b/README.qmd @@ -26,7 +26,11 @@ knitr::opts_chunk$set( The `{grumpy}` R package provides a way to read NumPy's `.npy` files into R. It supports a wide range of data types and array shapes. As a file format generated by a Python package, `.npy` files are prime candidates for using the `{reticulate}` package to read them into R. -However, this comes with downsides in terms of performance, flexibility, and robustness of the R package infrastructure. +However, using a Python package in R comes with downsides in terms of: +- performance, since the data needs to be copied across languages / sessions +- flexibility, since reticulate makes opinionated choices to work out of the box for most use cases +- robustness, since packages depending on reticulate need to risk breaking changes in the python packages they now depend on, or need to provide an environment via `{basilisk}`. + `{grumpy}`, on the other hand, is a pure R package with no dependency and is performant, flexible. Overall, it is designed to be used deep in the dependency graph of other packages. For more details on the motivation and design principles underpinning `{grumpy}`, see the dedicated vignette: `vignette("design", package = "grumpy")`. diff --git a/inst/extdata/test_structured.npy b/inst/extdata/test_structured.npy index e20a5d0..36ea7f4 100644 Binary files a/inst/extdata/test_structured.npy and b/inst/extdata/test_structured.npy differ diff --git a/inst/scripts/generate_test_data.py b/inst/scripts/generate_test_data.py index 98e23cb..f67a1bb 100644 --- a/inst/scripts/generate_test_data.py +++ b/inst/scripts/generate_test_data.py @@ -58,7 +58,7 @@ # Structured array dtype = np.dtype([("id", "int32"), ("value", "float64"), ("name", "U10")]) -data = np.array([(1, 3.14, "Alice"), (2, 2.71, "Bob"), (3, 1.62, "Charlie"), (4, 0.0, "Dave"), (5, -1.0, "Eve"), (6, 2.0, "Frank"), (7, 33.12, "Grace")], dtype=dtype) +data = np.array([(1, 3.14, "Alice"), (2, 2.71, "Bob"), (3, 1.62, "Charlie"), (4, 0.0, "Dave"), (5, -1.0, "Eve"), (6, 2.0, "Frank"), (7, 33.12, "Grace"), (8, 13.9, "Hugo")], dtype=dtype).reshape((2, 4)) np.save("inst/extdata/test_structured.npy", data) # NPZ archive (multiple arrays) diff --git a/man/convert_bytes_to_array.Rd b/man/convert_bytes_to_array.Rd index 073e40c..bb25480 100644 --- a/man/convert_bytes_to_array.Rd +++ b/man/convert_bytes_to_array.Rd @@ -9,8 +9,8 @@ convert_bytes_to_array(bytes, what, shape, size, endian) \arguments{ \item{bytes}{A raw vector containing the bytes to convert} -\item{what}{A character specifying the base type to convert to (e.g., \code{"float"}, -\code{"int"}, \code{"string"}, etc.)} +\item{what}{A character specifying the base type to convert to (e.g., +\code{"float"}, \code{"int"}, \code{"string"}, etc.)} \item{shape}{A numeric vector with desired shape of the output array} @@ -21,8 +21,7 @@ specified type} single-byte types)} } \value{ -An R array containing the converted data, with the specified shape and -data type. +An R array containing the converted data, with the specified shape } \description{ This is a replacement for \code{\link[=readBin]{readBin()}} that can handle the various data types @@ -33,7 +32,12 @@ x <- matrix(c(3L, 6L, 2L, 1L, 12L, 0L), nrow = 2, ncol = 3) x y <- writeBin(c(x), raw()) |> - convert_bytes_to_array("int", shape = c(2L, 3L), size = 4L, endian = "little") + convert_bytes_to_array( + "int", + shape = c(2L, 3L), + size = 4L, + endian = "little" + ) y dim(y) is.array(y) diff --git a/man/parse_npy_datatype.Rd b/man/parse_npy_datatype.Rd index 8aed71d..597627f 100644 --- a/man/parse_npy_datatype.Rd +++ b/man/parse_npy_datatype.Rd @@ -7,19 +7,26 @@ parse_npy_datatype(descr) } \arguments{ -\item{descr}{A NumPy dtype description string, or a list of such strings fo +\item{descr}{A NumPy dtype description string, or a list of such strings for structured dtypes} } \value{ -A list containing the parsed data type information, including the base -type, the number of bytes, and the endianness +A list containing the parsed data type information, including the +base type, the number of bytes, and the endianness } \description{ Parse a NumPy Array-protocol type strings } +\details{ +If a \code{list} is passed to \code{descr}, each element can be of length 1, or of +length 2 in which case the first element corresponds to the name of the field +and the second to its dtype. +} \examples{ parse_npy_datatype(">i8") parse_npy_datatype("|b1") +# A structured datatype where each element has 3 components, all integers, +# named "r", "g" and "b". parse_npy_datatype(list(c("r", " size ``` -**Without compression**, the equivalent Zarr data is thus 320 kB on disk, so `round(8e8 / size)` times smaller than the `.npy` file. We could also use compression to further reduce the size of the Zarr file on disk, but this is out of scope for this vignette. +**Without compression**, the equivalent Zarr data is thus 320 kB on disk, so `r round(8e8 / size)` times smaller than the `.npy` file. We could also use compression to further reduce the size of the Zarr file on disk, but this is out of scope for this vignette. ## Decoding speed comparison @@ -65,7 +65,7 @@ np$save(f_npy, x) bm <- bench::mark( grumpy = read_npy(f_npy), zarr = read_zarr_array(f_zarr), - iterations = 50 + iterations = 50L ) bm summary(bm, relative = TRUE) diff --git a/vignettes/features.qmd b/vignettes/compatibility.qmd similarity index 71% rename from vignettes/features.qmd rename to vignettes/compatibility.qmd index ed9c795..4513ea7 100644 --- a/vignettes/features.qmd +++ b/vignettes/compatibility.qmd @@ -1,5 +1,5 @@ --- -title: "Supported features" +title: "NumPy format compatibility" vignette: > %\VignetteIndexEntry{Supported features} %\VignetteEngine{quarto::html} @@ -10,15 +10,19 @@ vignette: > library(grumpy) ``` +## NumPy format versions + +[`.npy` format versions](https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html#version-numbering) 1.0, 2.0, and 3.0 (latest at the time of writing) as all supported. + ## Data types -The following data types are supported: +The following data types are supported (use `parse_npy_datatype()` to decode them in plain language): ```{r, echo = FALSE, results = "asis"} -types <- grumpy:::supported_types |> - as.list() |> +types <- grumpy:::supported_types |> + as.list() |> names() -types <- types[order(gsub("[<>|]", "", types))] +types <- types[order(gsub("[<>|]", "", types))] sprintf("- `%s`", types) |> cat(sep = "\n") ``` @@ -45,4 +49,8 @@ The following data types are not yet supported, because we are unsure of their u ## Dimensions -Any number of dimensions is supported, including zero-dimensional arrays (scalar values) and one-dimensional arrays (vectors). The shape of the original NumPy array is preserved in the output R object, and can be accessed with the `dim()` function. \ No newline at end of file +Any number of dimensions is supported, including zero-dimensional arrays (scalar values) and one-dimensional arrays (vectors). The shape of the original NumPy array is preserved in the output R object, and can be accessed with the `dim()` function. + +## Pickled objects + +Python object array created with `np.save(..., allow_pickle = True)` are not supported. diff --git a/vignettes/design.qmd b/vignettes/design.qmd index cf4b5c8..645136c 100644 --- a/vignettes/design.qmd +++ b/vignettes/design.qmd @@ -12,6 +12,22 @@ vignette: > Writing is out of scope. When working across multiple languages, one should prefer high-performance interoperable formats ([parquet](https://parquet.apache.org/), [Zarr](https://zarr-specs.readthedocs.io/en/latest/v3/core/), etc.). +# Dependencies + +Because `{grumpy}` is a intended to be used deep in the dependency graph of other packages, we want to minimize the number of dependencies. +There is currently no dependency, and no new dependency should be added unless it provides a significant performance improvement. + +# Function signatures + +## Inputs + +- The first argument of `read_npy()` and `read_npz()` is named `file` and is a connection or the path to the file to read. This is consistent with base R functions such as `read.csv()`, `readRDS()`, etc. + +## Outputs + +- The output of `read_npy()` is always an array (i.e., `is.array()` is `TRUE`), even for structured datatypes. This is to keep the output consistent and as conceptually close as possible to the original NumPy array. +- The output of `read_npz()` is always a named list of arrays, even if the `.npz` file contains only one array. A stable output type is important for downstream analysis. + # FAQ ## Why not use `reticulate`? @@ -19,4 +35,4 @@ Writing is out of scope. When working across multiple languages, one should pref - When reading `.npy` files with `{reticulate}`, at some point in time, two or three copies of the data are made in memory. This can be problematic for large files. With `{grumpy}`, two copies of the data are made in memory, with plans to make just one copy in the cases where the data type matches R native types. - Reading data with `{reticulate}` requires a Python installation and additional python packages, which users in restricted environments may not have access to. `{grumpy}` is a pure R package with no external dependencies. This is especially important as we expect `{grumpy}` to be used deep in the dependency graph of other packages, and we want to minimize the number of dependencies. -- A dedicated R package gives us more flexibility in how edge cases such as 64 bits integers are handled. `{reticulate}` automatically and silently converts 64 bits integers to double, which is a sensible default for many use cases. But we may want to have more control over this behavior, and `{grumpy}` will allow us to do that in the future. Another good example are structured data types (record arrays), which are returned as data.frames, not arrays, with `{reticulate}`. +- A dedicated R package gives us more flexibility in how edge cases such as 64-bit integers are handled. `{reticulate}` automatically and silently converts 64-bit integers to double, which is a sensible default for many use cases. But we may want to have more control over this behavior, and `{grumpy}` will allow us to do that in the future. Another good example are structured data types (record arrays), which are returned as `data.frame`s, not `array`s, with `{reticulate}`. diff --git a/vignettes/grumpy.qmd b/vignettes/grumpy.qmd index b7f2600..aed06e3 100644 --- a/vignettes/grumpy.qmd +++ b/vignettes/grumpy.qmd @@ -17,13 +17,16 @@ This package allows users to read a wide variety of `.npy` and `.npz` files in R We envision users may want to perform some steps of their data analysis in Python and others in R. -It is thus important to be able to read and write files in both languages. +It is thus important to be able to read and write files in both languages. Note however that `grumpy` does not support writing as we want to explicitly encourage users to use dedicated formats designed for interoperability. -Note however we would usually push users towards more advanced and performant formats such as [Zarr](https://zarr-specs.readthedocs.io/en/latest/v3/core/) for large datasets. Zarr datasets are supported for example by the `{Rarr}` Bioconductor package. +In particular, the following formats are designed for large datasets, high-performance, and partial or lazy reading, including on cloud storage: + +- if you are working with array-like data (the most likely use case for `.npy` files), we recommend using [Zarr](https://zarr-specs.readthedocs.io/en/latest/v3/core/) instead. Zarr datasets are supported by the `{Rarr}` Bioconductor package. +- if you are working with tabular data, we recommend using [Apache Arrow](https://arrow.apache.org/) instead. Arrow datasets are supported by the `{arrow}` CRAN package. ## Using grumpy -Most users are expected to mostly want to use `grumpy::read_npy()` and `grumpy::read_npz()` to read `.npy` and `.npz` files, respectively. These functions will return R objects that are equivalent to the original NumPy arrays, allowing users to easily manipulate and analyze the data in R. +Use `read_npy()` and `read_npz()` to read `.npy` and `.npz` files, respectively. These functions will return R objects that are equivalent to the original NumPy arrays, allowing users to easily manipulate and analyze the data in R. ```{r} read_npy(system.file("extdata", "test_2d.npy", package = "grumpy")) @@ -31,12 +34,27 @@ read_npy(system.file("extdata", "test_2d.npy", package = "grumpy")) ### Structured datatypes -One notable example are structured datatypes, where each element of the array is a record with named fields. +A more complex data structure is provided by structured datatypes, where each element of the array is a record with named fields. + To keep the output consistent and as conceptually close as possible to the original NumPy array, `grumpy` returns a list of list, with a `dim()` attribute to preserve the original shape of the array. It behaves like a standard R array, but each element is a list of the fields of the original structured datatype. -Note that in many cases, this is not efficient for any downstream analysis, and users may want to convert the output to a more standard R data structure such as a `data.frame` or `data.table` for easier manipulation. +```{r} +struct <- read_npy( + system.file("extdata", "test_structured.npy", package = "grumpy") +) +struct +dim(struct) +struct[[1L]] +``` +Note that in many cases, this is not efficient for any downstream analysis, and users may want to convert the output to a more standard R data structure such as a `data.frame` or `data.table` for easier manipulation. +```{r} +unlist(struct) |> + matrix(ncol = length(struct[[1L]]), byrow = TRUE) |> + as.data.frame() +``` +Doing so loses the original shape of the array, but it is unclear if this is a problem in practice, as structured datatypes are often used to store what should be tabular data.