Allow interior mutability for the context, read_until_with_ctx to allow ASTERIX "Extended Length Data Items"

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## [Unreleased]
 
+### Added
+
+- Changed the requirement that the context (`Ctx`) must be `Copy` in some calls.
+  It is changed to `Clone`, this **allows to add interior mutability** in the context
+  (like `Rc<Cell<bool>>`). This is no breaking change, since `Copy` implies `Clone`.
+- Added an attribute `until_with_ctx`: **like `until`, but it also gets the context**.
+- Added `writer_ctx` for fields: overrides `ctx` for the writer case - this allows
+  a **different context value for reader and writer** (the reader gets the original
+  `ctx`).
+- Added an attribute `read_post_processing`: code which is inserted after a field is
+  read (including a temp field). You have full context access (**and you can modify
+  the context there**).
 ## [0.20.3](https://github.com/sharksforarms/deku/compare/deku-v0.20.2...deku-v0.20.3) - 2026-01-15
 
 ### Fixed

deku-derive/src/lib.rs

@@ -583,18 +583,24 @@ struct FieldData {
     /// tokens providing the number of bytes for the length of the container
     bytes_read: Option<TokenStream>,
 
-    /// a predicate to decide when to stop reading elements into the container
+    /// a predicate to decide when to stop reading elements into the container (receives the current container element)
     until: Option<TokenStream>,
 
+    /// a predicate to decide when to stop reading elements into the container (receives the current container element and a clone of the context)
+    until_with_ctx: Option<TokenStream>,
+
     /// read until `reader.end()`
     read_all: bool,
 
     /// apply a function to the field after it's read
     map: Option<TokenStream>,
 
-    /// context passed to the field
+    /// context passed to the field (reader/writer; see writer_ctx)
     ctx: Option<Punctuated<syn::Expr, syn::token::Comma>>,
 
+    /// writer_context passed to the field (overrides ctx)
+    writer_ctx: Option<Punctuated<syn::Expr, syn::token::Comma>>,
+
     /// map field when updating struct
     update: Option<TokenStream>,
 
@@ -627,6 +633,9 @@ struct FieldData {
     /// write given value of temp field
     temp_value: Option<TokenStream>,
 
+    /// post_process
+    read_post_processing: Option<TokenStream>,
+
     /// default value code when used with skip or cond
     default: Option<TokenStream>,
 
@@ -678,8 +687,10 @@ impl FieldData {
         any_option_set = any_option_set
             || self.bytes_read.is_some()
             || self.until.is_some()
+            || self.until_with_ctx.is_some()
             || self.map.is_some()
             || self.ctx.is_some()
+            || self.writer_ctx.is_some()
             || self.update.is_some()
             || self.reader.is_some()
             || self.writer.is_some();
@@ -700,6 +711,7 @@ impl FieldData {
         any_option_set = any_option_set
             || self.pad_bytes_after.is_some()
             || self.temp_value.is_some()
+            || self.read_post_processing.is_some()
             || self.cond.is_some()
             || self.assert.is_some()
             || self.assert_eq.is_some()
@@ -720,6 +732,11 @@ impl FieldData {
             .map(|s| s.parse_with(Punctuated::parse_terminated))
             .transpose()
             .map_err(|e| e.to_compile_error())?;
+        let writer_ctx = receiver
+            .writer_ctx?
+            .map(|s| s.parse_with(Punctuated::parse_terminated))
+            .transpose()
+            .map_err(|e| e.to_compile_error())?;
 
         let data = Self {
             ident: receiver.ident,
@@ -733,9 +750,11 @@ impl FieldData {
             bits_read: receiver.bits_read?,
             bytes_read: receiver.bytes_read?,
             until: receiver.until?,
+            until_with_ctx: receiver.until_with_ctx?,
             read_all: receiver.read_all,
             map: receiver.map?,
             ctx,
+            writer_ctx,
             update: receiver.update?,
             reader: receiver.reader?,
             writer: receiver.writer?,
@@ -748,6 +767,7 @@ impl FieldData {
             pad_bytes_after: receiver.pad_bytes_after?,
             temp: receiver.temp,
             temp_value: receiver.temp_value?,
+            read_post_processing: receiver.read_post_processing?,
             default: receiver.default?,
             cond: receiver.cond?,
             assert: receiver.assert?,
@@ -824,12 +844,13 @@ impl FieldData {
         #[cfg(feature = "bits")]
         if data.read_all
             && (data.until.is_some()
+                || data.until_with_ctx.is_some()
                 || data.count.is_some()
                 || (data.bits_read.is_some() || data.bytes_read.is_some()))
         {
             return Err(cerror(
                 data.read_all.span(),
-                "conflicting: `read_all` cannot be used with `until`, `count`, `bits_read`, or `bytes_read`",
+                "conflicting: `read_all` cannot be used with `until`, `until_with_ctx`, `count`, `bits_read`, or `bytes_read`",
             ));
         }
 
@@ -1107,6 +1128,10 @@ struct DekuFieldReceiver {
     #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
     until: Result<Option<TokenStream>, ReplacementError>,
 
+    /// a predicate to decide when to stop reading elements into the container
+    #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
+    until_with_ctx: Result<Option<TokenStream>, ReplacementError>,
+
     /// read until `reader.end()`
     #[darling(default)]
     read_all: bool,
@@ -1115,13 +1140,20 @@ struct DekuFieldReceiver {
     #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
     map: Result<Option<TokenStream>, ReplacementError>,
 
-    /// context passed to the field.
+    /// context passed to the field (reader/writer). See writer_ctx.
     /// A comma separated argument list.
     // TODO: The type of it should be `Punctuated<Expr, Comma>`
     //       https://github.com/TedDriggs/darling/pull/98
     #[darling(default = "default_res_opt", map = "map_option_litstr")]
     ctx: Result<Option<syn::LitStr>, ReplacementError>,
 
+    /// writer_context passed to the field. Overrides ctx.
+    /// A comma separated argument list.
+    // TODO: The type of it should be `Punctuated<Expr, Comma>`
+    //       https://github.com/TedDriggs/darling/pull/98
+    #[darling(default = "default_res_opt", map = "map_option_litstr")]
+    writer_ctx: Result<Option<syn::LitStr>, ReplacementError>,
+
     /// map field when updating struct
     #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
     update: Result<Option<TokenStream>, ReplacementError>,
@@ -1164,6 +1196,10 @@ struct DekuFieldReceiver {
     #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
     temp_value: Result<Option<TokenStream>, ReplacementError>,
 
+    /// post process read values
+    #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
+    read_post_processing: Result<Option<TokenStream>, ReplacementError>,
+
     /// default value code when used with skip
     #[darling(default = "default_res_opt", map = "map_litstr_as_tokenstream")]
     default: Result<Option<TokenStream>, ReplacementError>,

deku-derive/src/macros/deku_read.rs

@@ -671,11 +671,14 @@ fn emit_field_read(
         &f.bits_read,
         &f.bytes_read,
         &f.until,
+        &f.until_with_ctx,
         &f.cond,
         &f.default,
         &f.map,
+        &f.read_post_processing,
         &f.reader,
         &f.ctx.as_ref().map(|v| quote!(#v)),
+        &f.writer_ctx.as_ref().map(|v| quote!(#v)),
         &f.assert,
         &f.assert_eq,
     ];
@@ -766,6 +769,13 @@ fn emit_field_read(
         quote! {}
     };
 
+    let read_post_processing = if f.read_post_processing.is_some() {
+        let code = &f.read_post_processing.clone().unwrap();
+        code.clone()
+    } else {
+        quote! {}
+    };
+
     let magic_read = if let Some(magic) = &f.magic {
         emit_magic_read_lit(&crate_, magic)
     } else {
@@ -876,6 +886,16 @@ fn emit_field_read(
                     (::#crate_::ctx::Limit::new_until(#field_until), (#read_args))
                 )?
             }
+        } else if let Some(field_until_with_ctx) = &f.until_with_ctx {
+            // We wrap the input into another closure here to enforce that it is actually a callable
+            // Otherwise, an incorrectly passed-in integer could unexpectedly convert into a `Count` limit
+            quote! {
+                #type_as_deku_read::from_reader_with_ctx
+                (
+                    __deku_reader,
+                    (::#crate_::ctx::Limit::new_until_with_ctx(#field_until_with_ctx), (#read_args))
+                )?
+            }
         } else if f.read_all {
             quote! {
                 {
@@ -994,6 +1014,7 @@ fn emit_field_read(
         };
         let #field_ident = &#internal_field_ident;
 
+        #read_post_processing
         #field_assert
         #field_assert_eq
 

deku-derive/src/macros/deku_write.rs

@@ -696,6 +696,7 @@ fn emit_field_write(
         &f.writer,
         &f.cond,
         &f.ctx.as_ref().map(|v| quote!(#v)),
+        &f.writer_ctx.as_ref().map(|v| quote!(#v)),
         &f.assert,
         &f.assert_eq,
     ];
@@ -747,7 +748,11 @@ fn emit_field_write(
             #[cfg(not(feature = "bits"))]
             None,
             f.bytes.as_ref(),
-            f.ctx.as_ref(),
+            if f.writer_ctx.is_some() {
+                f.writer_ctx.as_ref()
+            } else {
+                f.ctx.as_ref()
+            },
             field_bit_order,
         )?;
 

examples/many.rs

@@ -23,7 +23,7 @@ fn main() {
 
     let mut binding = Cursor::new(custom.clone());
     let mut reader = Reader::new(&mut binding);
-    let ret = <Vec<Test> as DekuReader<Limit<_, _>>>::from_reader_with_ctx(
+    let ret = <Vec<Test> as DekuReader<Limit<_, _, _, _>>>::from_reader_with_ctx(
         &mut reader,
         Limit::new_count(10_0000),
     );

src/attributes.rs

@@ -47,7 +47,9 @@ enum DekuEnum {
 | [bits_read](#bits_read) | field | Set the field representing the number of bits to read into a container
 | [bytes_read](#bytes_read) | field | Set the field representing the number of bytes to read into a container
 | [until](#until) | field | Set a predicate returning when to stop reading elements into a container
+| [until_with_ctx](#context_with_interior_mutability) | field | like `until`, but the predicate also receives the context as second argument
 | [read_all](#read_all) | field | Read until [reader.end()] returns `true`
+| [read_post_processing](#context_with_interior_mutability) | field | Code to postprocess read data (including temp values; can be used to modify the context via interior mutability)
 | [update](#update) | field | Apply code over the field when `.update()` is called
 | [temp](#temp) | field | Read the field but exclude it from the struct/enum
 | [temp_value](#temp_value) | field | Write the field but exclude it from the struct/enum
@@ -61,7 +63,8 @@ enum DekuEnum {
 | [map](#map) | field | Specify a function or lambda to apply to the result of the read
 | [reader](#readerwriter) | variant, field | Custom reader code
 | [writer](#readerwriter) | variant, field | Custom writer code
-| [ctx](#ctx) | top-level, field| Context list for context sensitive parsing
+| [ctx](#ctx) | top-level, field| Context list for context sensitive parsing (reader/writer)
+| [writer_ctx](#context_with_interior_mutability) | top-level, field| Context list for context sensitive parsing (overrides ctx for the writer part)
 | [ctx_default](#ctx_default) | top-level, field| Default context values
 | enum: [id](#id) | top-level, variant | enum or variant id value
 | enum: [id_endian](#id_endian) | top-level | Endianness of *just* the enum `id`
@@ -1154,6 +1157,123 @@ assert_eq!(&*data, value);
 # fn main() {}
 ```
 
+# <div id="context_with_interior_mutability">Context with Interior Mutability (using `until_with_ctx`, `writer_ctx`, `read_post_processing`)</div>
+
+The following example demonstrates how to read and write an array of data, where a flag in each data item (`auto_fx`) indicates whether that item is the last in a containing array.
+**Importantly, this flag is not part of the actual data structure** presented to the user and is only a temporary construct used during reading and writing.
+This ensures the flag is always set correctly and cannot be misused (e.g. be forgotten to be set correctly).
+
+The central idea is to use a **context with interior mutability** (specifically, the `fx` field) to make the temporary value available to the outer structure while traversing the array. This context can also be used to increment a counter representing the current index of the traversed array.
+
+**Key Steps Illustrated in This Example:**
+
+1. **Define a context with interior mutability**:
+   1. A mutable `idx` field representing the **current index of the array** (mutable through a `Cell`, shared through an `Rc`).
+   2. A mutable flag `fx` indicating the **end of the array** (mutable through a `Cell`, shared through an `Rc`).
+   3. A (non-mutable) value `n` representing the **length of the array** (while writing) or zero (while reading).
+
+2. **Set up the context separately for reading (`ctx`) and writing (`writer_ctx`)**. This is
+   important, since the final array length is unknown while reading.
+
+3. **During reading**:
+   1. The temporary information stored in the `auto_fx` field must be passed back to the caller via the context.
+   2. `read_post_processing` stores the temporary value in the context.
+   3. `until_with_ctx` in the surrounding array uses this context to control how much data to read (until the `fx` flag is zero).
+
+4. **During writing**:
+   - The index is updated in each step and the `auto_fx` field is set accordingly (last entry is set to `false`).
+
+Example:
+
+* Here, the user fills an array with x/y data
+  ```rust,ignore
+  A { items: vec![
+     B { x: 8, y: 9 },
+     B { x: 7, y: 9 },
+     B { x: 6, y: 9 },
+  ] };
+  ```
+* This data is stored on disk as x/y/"index-in-array"/"fx=0 for the last element, else fx=1"
+  ```rust,ignore
+             vec![8, 9, 0, 1, 7, 9, 1, 1, 6, 9, 2, 0]
+  //                    ^  ^        ^  ^        ^  ^
+  //                    |  fx=1     |  fx=1     |  fx=0 (last)
+  //                   idx=0      idx=1       idx=2
+  ```
+* The user (providing the x/y data) is not aware of the index and the `fx` field
+  stored on disk (and cannot break the binary data format by providing some wrong data).
+
+* Note: this enables to implement "Extended Length Data Items" from the
+  [ASTERIX EUROCONTOL](https://www.eurocontrol.int/publication/eurocontrol-specification-surveillance-data-exchange-part-i)
+  standard, without making control flags (`fx`) visible in the user data.
+
+```rust
+# #[cfg(feature = "alloc")]
+# use deku::prelude::*;
+# #[cfg(feature = "alloc")]
+# fn main() {
+#[derive(Debug, Clone)]
+struct IndexContext {
+    idx: std::rc::Rc<std::cell::Cell<usize>>, // see text, 1.1
+    n: usize,                                 // see text, 1.2
+    fx: std::rc::Rc<std::cell::Cell<bool>>,   // see text, 1.3
+}
+#[deku_derive(DekuRead, DekuWrite)]
+#[derive(PartialEq, Debug, Clone)]
+struct A {
+    #[deku(
+        until_with_ctx = "|_:&B,ctx:IndexContext| !ctx.fx.get()", // see text, 3.1 + 3.3
+
+        // see text, 2
+        ctx = "IndexContext { idx: std::rc::Rc::new(std::cell::Cell::new(0)), n: 0, fx: std::rc::Rc::new(std::cell::Cell::new(false))}",
+        writer_ctx = "IndexContext { idx: std::rc::Rc::new(std::cell::Cell::new(0)), n: items.len(), fx: std::rc::Rc::new(std::cell::Cell::new(false)) }"
+    )]
+    items: Vec<B>,
+}
+#[deku_derive(DekuRead, DekuWrite)]
+#[derive(PartialEq, Debug, Clone)]
+#[deku(
+    ctx = "ctx: IndexContext",
+    ctx_default = "IndexContext{idx: std::rc::Rc::new(std::cell::Cell::new(0)), n: 0, fx: std::rc::Rc::new(std::cell::Cell::new(false))}"
+)] // this struct uses a context for serialization. For deserialization it also works with the default context.
+struct B {
+    x: u8,
+    y: u8,
+    #[deku(
+        temp,
+        temp_value = "{let ret = ctx.idx.get() as u8; ctx.idx.set(ctx.idx.get()+1); ret}"
+    )]
+    idx_automatically_filled: u8,
+    #[deku(
+        read_post_processing = "ctx.fx.set(*auto_fx!=0);", // see text, 3.2 + 3.3
+        temp,
+        temp_value = "if ctx.idx.get() < ctx.n {1} else {0}" // see text, 4
+    )]
+    auto_fx: u8,
+}
+#
+#     let test_data = A {
+#         items: vec![B { x: 8, y: 9 }, B { x: 7, y: 9 }, B { x: 6, y: 9 }],
+#     };
+#
+#     let ret_write: Vec<u8> = test_data.clone().try_into().unwrap();
+#     assert_eq!(vec![8, 9, 0, 1, 7, 9, 1, 1, 6, 9, 2, 0], ret_write);
+#     //                    ^  ^        ^  ^        ^  ^
+#     //                    |  fx=1     |  fx=1     |  fx=0 (last)
+#     //                   idx=0      idx=1       idx=2
+#
+#     // read the data back
+#     let check_data = A::from_bytes((&ret_write, 0)).unwrap().1;
+#     assert_eq!(check_data, test_data);
+#
+#     // check with fx=0 after the second element:
+#     let check_data = A::from_bytes((&[8, 9, 0, 1, 7, 9, 1, 0], 0)).unwrap().1;
+#     assert_eq!(check_data.items.len(), 2);
+# }
+# #[cfg(not(feature = "alloc"))]
+# fn main() {}
+```
+
 # update
 
 Specify custom code to run on the field when `.update()` is called on the struct/enum