Ndarray Xarray Feature branch

CHANGELOG.md

@@ -9,13 +9,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
-- Normalised `hash_element_at` so NaN maps to a dummy/sentinel value
+### Added
+
+**NdArray, XArray and DLPack Feature**:
+- `NdArray` n-dimensional dense array (`ndarray` feature) over `Buffer<T>`,
+  generic over f32/f64, with NaN null semantics, `NdArrayV` zero-copy views,
+  transpose and axis permutation, and Table/Matrix/Array interop.
+- `SuperNdArray` chunked n-dimensional array with `SuperNdArrayV`
+  chunk-spanning views and rechunking.
+- `XArray` labelled n-dimensional array (`xarray` feature) with named
+  dimensions, coordinate selection via `at`/`between`/`nearest`, and owned,
+  view, or chunked storage behind one type.
+- DLPack tensor interchange (`dlpack` feature) over the legacy and 1.x
+  versioned ABIs for zero-copy sharing with PyTorch, JAX, and TensorFlow.
+- `AxisSelection` trait for `.s()` axis slicing, with `NdArrayVT`,
+  `SuperNdArrayVT`, and `XArrayVT` view tuple aliases.
+- Broadcast kernels for `NdArray`, `SuperNdArray`, and `XArray`, plus
+  `Value` variants, conversions, and concatenation for the new types.
+- `NdArray` in minarrow-py with the DLPack protocol - `__dlpack__`,
+  `from_dlpack`, and named bridges to NumPy, PyTorch, JAX, TensorFlow,
+  and CuPy.
+- `PyNdArray` in minarrow-pyo3 (`ndarray` feature) for Rust extensions
+  bridging tensors to Python, with the DLPack capsule glue hosted in
+  its `ffi::dlpack` and shared by minarrow-py.
+
+**Minor Feature Additions**
 - At `value_at` to `Array` and `ArrayV` for (opt-in) normalised equality checking.
-- Normalised equality checking for `Scalar` -0.0 == 0.0 and NaN == NaN.
-- Fixed null handling bug on String and Categorical set_str method.
+- Added bitmask-driven gather to Array, Table and their view variants.
+- Added default SIMD chipset behaviour.
+- Added bitwise kernels.
+- Added default display trait impl to `Scalar`.
+- Added `LBuffer::freeze()`.
+- Added `NdArray` n-dimensional array with views and chunked variants.
+- Added `XArray` labelled n-dimensional array.
+- Added axis selection, broadcasting, and `Value` support for the n-dimensional types.
+- Added DLPack FFI for zero-copy PyTorch, JAX, and TensorFlow interchange, with `NdArray` in minarrow-py and minarrow-pyo3.
 - Added `get`, `get_unchecked`, `get_str` and `get_str_unchecked` methods to `Array`
 - Added bitmask gather to Array, Table and their view variants.
 
+### Bugfixes
+- Fixed a kernel branching issue in the arithmetic SIMD paths.
+- Fixed out-of-range categorical codes to resolve to the empty string.
+- Fixed `StringArray` length reporting the byte length when `MaskedArray` was not imported.
+- Fixed null handling bug on String and Categorical set_str method.
+
+### Modified
+- Bumped pyo3 to 0.29.
+- Normalised `hash_element_at` so NaN maps to a dummy/sentinel value
+- Normalised equality checking for `Scalar` -0.0 == 0.0 and NaN == NaN.
 ## [0.15.0] - 2026-06-30
 
 ### Added

Cargo.toml

@@ -138,6 +138,24 @@ views = []
 # with large data sizes.
 matrix = []
 
+# N-dimensional f32/f64 container with compact column-major layout.
+# Slicing and axis selection are available through the `select` feature.
+# Useful for bridging into Python ML/AI runtimes via DLPack.
+#
+# Note that this does not include statistical routines.
+ndarray = []
+
+# DLPack tensor interchange with Python and other runtimes. Compatible
+# ownership, alignment, layout, and protocol choices allow direct sharing.
+# Requires the `ndarray` feature.
+dlpack = ["ndarray"]
+
+# Labelled N-dimensional array wrapping NdArray with named dimensions
+# and optional coordinate labels per axis. Requires `ndarray` feature.
+# Coordinate value lookup is available when `scalar_type`, `views`, and
+# `select` are also enabled.
+xarray = ["ndarray"]
+
 # Adds `to_apache_arrow()` for casting into that library.
 cast_arrow = ["arrow", "arrow-schema"]
 
@@ -238,7 +256,7 @@ fast_dict = [
     "dep:parking_lot",
 ]
 
-# Adds typed arithmetic broadcasting for add, sub, mult, div, rem
+# Adds typed arithmetic broadcasting for add, sub, mult, div, rem.
 broadcast = []
 
 # Adds Hash and Eq implementations for Scalar, and hash_element_at for Array.
@@ -312,33 +330,49 @@ rustdoc-args = ["--cfg", "docsrs"]
 [[example]]
 name = "arithmetic"
 path = "examples/arithmetic.rs"
-required-features = ["broadcast"]
+required-features = ["broadcast", "value_type"]
 
 # Broadcasting examples
 [[example]]
 name = "test_broadcasting"
 path = "examples/broadcasting/test_broadcasting.rs"
-required-features = ["broadcast"]
+required-features = ["broadcast", "value_type"]
 
 [[example]]
 name = "test_scalar_arithmetic"
 path = "examples/broadcasting/test_scalar_arithmetic.rs"
-required-features = ["broadcast", "scalar_type"]
+required-features = ["broadcast", "value_type", "scalar_type"]
 
 [[example]]
 name = "test_string_broadcasting"
 path = "examples/broadcasting/test_string_broadcasting.rs"
-required-features = ["broadcast"]
+required-features = ["broadcast", "value_type"]
 
 [[example]]
 name = "test_value_ops"
 path = "examples/broadcasting/test_value_ops.rs"
-required-features = ["broadcast"]
+required-features = ["broadcast", "value_type"]
 
 [[example]]
 name = "test_value_macros"
 path = "examples/broadcasting/test_value_macros.rs"
-required-features = ["broadcast"]
+required-features = ["broadcast", "value_type"]
+
+# N-dimensional examples
+[[example]]
+name = "ndarray"
+path = "examples/ndarray.rs"
+required-features = ["ndarray"]
+
+[[example]]
+name = "super_ndarray"
+path = "examples/super_ndarray.rs"
+required-features = ["ndarray"]
+
+[[example]]
+name = "xarray"
+path = "examples/xarray.rs"
+required-features = ["xarray", "scalar_type", "views", "select"]
 
 [[bench]]
 name = "hotloop_benchmark_simd"

README.md

@@ -93,6 +93,8 @@ Semantic groupings (`NumericArray`, `TextArray`, `TemporalArray`) support flexib
 
 Bonus LAPACK-compatible `Matrix` and `Cube` types support analytical workload variations.
 
+`NdArray` n-dimensional numeric data, with chunked (`SuperNdArray`) and labelled (`XArray`) forms, and hands tensors to PyTorch, JAX, and NumPy zero-copy over DLPack.
+
 ### Zero-Copy Views
 
 Ergonomic zero-copy row and column selection.
@@ -225,7 +227,7 @@ Minarrow's direct access is within the noise threshold of raw Vec performance wh
 - **Rust to Python¹** 1m rows, 2 columns : **165ns**
 - **Python to Rust¹** 1m rows, 2 columns:  **2.5μs**
 
-See `minarrow-pyo3/examples` 
+See `minarrow-pyo3/examples`
 
 ### Test machine
 ▎ ¹ = Intel Core Ultra 7 155H · 32 GB · Ubuntu 24.04 · 1.97-nightly release build.
@@ -262,6 +264,9 @@ Additional types:
 | `value_type` | Catch-all `Value` enum for unified typing |
 | `matrix` | 2D matrix with BLAS/LAPACK-compatible layout |
 | `cube` | Stacks tables along an extra axis |
+| `ndarray` | N-dimensional array for bridging into ML & AI, with a chunked version and native DLPack interchange |
+| `xarray` | N-dimensional array with named dims and coordinates for spatial queries |
+| `dlpack` | DLPack tensor interchange with PyTorch, JAX, TensorFlow |
 | `shared_dict` | Shared source of truth for categorical dictionaries. |
 
 Performance:

examples/ndarray.rs

@@ -0,0 +1,71 @@
+// Copyright 2025 Peter Garfield Bower
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use minarrow::traits::selection::RowSelection;
+use minarrow::{Concatenate, NdArray, nd};
+
+fn main() {
+    // Flat input follows NdArray's column-major layout.
+    let a = NdArray::from_slice(
+        &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0],
+        &[4, 3],
+    );
+
+    println!("=== Shape and access ===\n");
+    println!("shape: {:?}", a.shape());
+    println!("strides: {:?}", a.strides());
+    println!("n_obs: {}", a.n_obs());
+    println!("a.get(&[2, 1]) = {}\n", a.get(&[2, 1]));
+
+    // Constructors create ordinary NdArray containers.
+    println!("=== Constructors ===\n");
+    let steps = NdArray::<f64>::linspace(0.0, 1.0, 5);
+    println!("linspace(0.0, 1.0, 5): {:?}", steps.as_slice());
+    let identity = NdArray::<f64>::eye(3);
+    println!("eye(3) diagonal: {:?}\n", (0..3).map(|i| identity.get(&[i, i])).collect::<Vec<_>>());
+
+    // Views can change shape, offsets, or strides without copying the data.
+    println!("=== Views ===\n");
+    println!("a.obs(1) - one observation as a 1D view");
+    let row = a.obs(1);
+    println!("  shape {:?}, values {:?}\n", row.shape(), (0..3).map(|j| row.get(&[j])).collect::<Vec<_>>());
+
+    println!("a.slice(nd![1..3, 0..2]) - range on both axes");
+    let window = a.slice(nd![1..3, 0..2]);
+    println!("  shape {:?}, window[0, 0] = {}\n", window.shape(), window.get(&[0, 0]));
+
+    println!("a.r(0..2) - axis-0 selection");
+    let head = a.r(0..2);
+    println!("  shape {:?}\n", head.shape());
+
+    println!("view.transpose() - zero-copy stride swap");
+    let t = a.as_view().transpose();
+    println!("  shape {:?}, t.get(&[1, 2]) = {}\n", t.shape(), t.get(&[1, 2]));
+
+    // Apply operation to each array element.
+    println!("=== Apply ===\n");
+    let doubled = a.apply(|v| v * 2.0);
+    println!("a.apply(|v| v * 2.0): doubled[3, 2] = {}\n", doubled.get(&[3, 2]));
+
+    // Concatenation extends axis 0 while retaining the trailing shape.
+    println!("=== Concatenate ===\n");
+    let b = NdArray::from_slice(&[13.0, 14.0, 15.0], &[1, 3]);
+    let joined = a.clone().concat(b).unwrap();
+    println!("a.concat(b): shape {:?}, last row starts with {}\n", joined.shape(), joined.get(&[4, 0]));
+
+    // A 2D conversion produces one Table column per axis-1 entry.
+    println!("=== To Table ===\n");
+    let table = a.clone().to_table(None).unwrap();
+    println!("{}", table);
+}

examples/super_ndarray.rs

@@ -0,0 +1,59 @@
+// Copyright 2025 Peter Garfield Bower
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use minarrow::structs::chunked::super_array::RechunkStrategy;
+use minarrow::{Consolidate, NdArray, SuperNdArray};
+
+fn main() {
+    // Simulate three sensor batches arriving with different observation
+    // counts. Every batch has two measurements per observation.
+    let mut snd = SuperNdArray::new("sensor_frames");
+    snd.push(NdArray::from_slice(&[1.0, 2.0, 10.0, 20.0], &[2, 2]));
+    snd.push(NdArray::from_slice(&[3.0, 4.0, 5.0, 30.0, 40.0, 50.0], &[3, 2]));
+    snd.push(NdArray::from_slice(&[6.0, 60.0], &[1, 2]));
+
+    println!("=== Shape across batches ===\n");
+    println!("n_batches: {}", snd.n_batches());
+    println!("n_obs: {}", snd.n_obs());
+    println!("shape: {:?}\n", snd.shape());
+
+    // Global indices address the combined observation range across batches.
+    println!("=== Global access ===\n");
+    println!("snd.get(&[0, 0]) = {} (batch 0)", snd.get(&[0, 0]));
+    println!("snd.get(&[3, 1]) = {} (batch 1)", snd.get(&[3, 1]));
+    println!("snd.get(&[5, 1]) = {} (batch 2)\n", snd.get(&[5, 1]));
+
+    // The requested observation window starts in one batch and ends in the next.
+    println!("=== Batch-spanning window ===\n");
+    let window = snd.slice(1, 3);
+    println!("snd.slice(1, 3): n_obs {}, spans {} slices", window.n_obs(), window.n_slices());
+    println!("window.get(&[1, 0]) = {}\n", window.get(&[1, 0]));
+
+    // Consolidate only when a consumer requires one contiguous allocation.
+    println!("=== Consolidate ===\n");
+    let flat = snd.clone().consolidate();
+    println!("consolidated shape: {:?}", flat.shape());
+    println!("flat.get(&[5, 1]) = {}\n", flat.get(&[5, 1]));
+
+    // Change the batch boundaries without changing the logical values.
+    println!("=== Rechunk ===\n");
+    let mut even = snd.clone();
+    even.rechunk(RechunkStrategy::Count(2)).unwrap();
+    println!("rechunk(Count(2)): n_batches {}, first batch obs {}", even.n_batches(), even.batch(0).unwrap().shape()[0]);
+
+    // Batch boundaries do not affect logical equality.
+    println!("\n=== Logical equality ===\n");
+    let single = SuperNdArray::from_batches(vec![flat], "sensor_frames");
+    println!("snd == consolidated single batch: {}", snd == single);
+}

examples/xarray.rs

@@ -0,0 +1,66 @@
+// Copyright 2025 Peter Garfield Bower
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use minarrow::{NdArray, XArray, arr_f64, arr_str32};
+
+fn main() {
+    // Four hourly observations of three measurements.
+    let data = NdArray::from_slice(
+        &[
+            20.1, 20.4, 20.9, 21.3, // temp
+            1.01, 1.02, 1.00, 0.99, // pressure
+            55.0, 54.0, 52.0, 51.0, // humidity
+        ],
+        &[4, 3],
+    );
+    let mut xa = XArray::new(data, &["hour", "measurement"]);
+    xa.assign_coords("hour", arr_f64![0.0, 1.0, 2.0, 3.0]);
+
+    println!("=== Dims and coords ===\n");
+    println!("{:?}", xa);
+    println!("dim_names: {:?}", xa.dim_names());
+    println!("shape: {:?}\n", xa.shape());
+
+    // Coordinate selection uses the values attached to the named axis.
+    println!("=== Coordinate selection ===\n");
+
+    println!("xa.at(\"hour\", 2.0) - collapse to one observation");
+    let at_two = xa.at("hour", 2.0);
+    println!("  shape {:?}, temp = {}\n", at_two.shape(), at_two.get(&[0]));
+
+    println!("xa.between(\"hour\", 1.0, 3.0) - inclusive window");
+    let window = xa.between("hour", 1.0, 3.0);
+    println!("  shape {:?}, first temp = {}\n", window.shape(), window.get(&[0, 0]));
+
+    println!("xa.nearest(\"hour\", 1.8) - closest coordinate");
+    let near = xa.nearest("hour", 1.8);
+    println!("  shape {:?}, temp = {}\n", near.shape(), near.get(&[0]));
+
+    // Positional selection can also address an axis by name.
+    println!("=== Named positional selection ===\n");
+    let first_two = xa.select(&[("hour", &(0..2))]);
+    println!("xa.select(&[(\"hour\", &(0..2))]): shape {:?}\n", first_two.shape());
+
+    // Transpose reorders data, dimension names, and coordinates together.
+    println!("=== Transpose ===\n");
+    let t = xa.transpose(&["measurement", "hour"]).unwrap();
+    println!("dims after transpose: {:?}, shape {:?}\n", t.dim_names(), t.shape());
+
+    // Axis-1 coordinate labels become Table column names.
+    println!("=== To Table ===\n");
+    let mut named = xa.clone();
+    named.assign_coords("measurement", arr_str32!(&["temp", "pressure", "humidity"]));
+    let table = named.to_table().unwrap();
+    println!("{}", table);
+}