diff --git a/bindings/c/clay_c.cpp b/bindings/c/clay_c.cpp index f8e62de3..8624fdc5 100644 --- a/bindings/c/clay_c.cpp +++ b/bindings/c/clay_c.cpp @@ -1121,6 +1121,50 @@ clay_result from_io(const io::IoStatus& s) { } } +// The append log's reaches as a binary tree of unions over index ranges: "does +// any append from index `from` on reach this box?" in about log(n) box tests +// instead of n (#665). A region invalidation asks it once per lagging seed, and +// a straight scan made that seeds x appends: 43 ms at 3,000 dabs against 0.3 ms +// before the check existed, growing with the length of the stroke. Consecutive +// dabs lie close together, so a range's union stays tight and prunes well. +// +// Built once per region invalidation, from a log that invalidation then +// forgets, so nothing keeps it in step with later appends. +class ReachTree { + public: + explicit ReachTree(std::span reaches) : count_(reaches.size()) { + while (width_ < count_) width_ *= 2; + nodes_.resize(2 * width_); // the padding leaves stay empty: they reach nothing + std::copy(reaches.begin(), reaches.end(), + nodes_.begin() + static_cast(width_)); + for (std::size_t k = width_ - 1; k >= 1; --k) { + nodes_[k] = nodes_[2 * k]; + nodes_[k].expand(nodes_[2 * k + 1]); + } + } + + // Whether any reach at index `from` or later intersects `box`. + bool any_from(std::size_t from, const math::Aabb& box) const { + return from < count_ && any_in(1, 0, width_, from, box); + } + + private: + // A node's union also holds entries before `from`, so it can only PRUNE; + // the answer is decided at a leaf, and every leaf reached here is at or + // after `from`. + bool any_in(std::size_t node, std::size_t lo, std::size_t hi, std::size_t from, + const math::Aabb& box) const { + if (hi <= from || !nodes_[node].intersects(box)) return false; + if (node >= width_) return true; + const std::size_t mid = lo + (hi - lo) / 2; + return any_in(2 * node, lo, mid, from, box) || any_in(2 * node + 1, mid, hi, from, box); + } + + std::size_t count_ = 0; + std::size_t width_ = 1; + std::vector nodes_; +}; + } // namespace struct clay_document; @@ -1379,7 +1423,13 @@ struct clay_document { // sitting at revision R wants entries from `R - append_base_` onward, and // several readers at different revisions can each take their own tail of // the same log. `appends_since` is that lookup. - void touch_appended(scene::LayerId layer, scene::NodeId node) { + // + // `reach` is what the append can change -- the command_influence_bound + // apply_edit took on both sides of it. The log keeps one per entry so that + // a region invalidation arriving before any refill can tell which of the + // seeds this append left behind it may still carry forward (#665; see + // `carried_across_appends`). + void touch_appended(scene::LayerId layer, scene::NodeId node, const math::Aabb& reach) { std::lock_guard lock(cache_mutex_); const std::uint64_t now = revision.load(std::memory_order_relaxed); // The log is only usable while it is CONTIGUOUS: it describes the @@ -1391,8 +1441,13 @@ struct clay_document { append_layer_ = layer; append_base_ = now; append_log_.clear(); + append_reach_.clear(); } append_log_.push_back(node); + // Widened to the whole of space when infinite at all, so the union + // tree a region edit builds over these (ReachTree) can never prune an + // entry the per-entry test would have counted as reaching. + append_reach_.push_back(reach.is_infinite() ? math::Aabb::infinite() : reach); // An append IS an order change: it grows the root list, so every // ordinal taken before it counts a different tail. This does not gate // the append fast path -- plan_resume never reads structure_revision_ @@ -2348,9 +2403,9 @@ struct clay_document { // against 0.16x for the same drag unmirrored). void touch_regions(std::span changed) { std::lock_guard lock(cache_mutex_); - forget_appends(); // not an append; no prefix may be reused const std::uint64_t next = revision.fetch_add(1, std::memory_order_relaxed) + 1; touch_region_locked(changed, kFrontierDrop, next); + forget_appends(); // not an append; no prefix may be reused } // The region invalidation for an edit that MOVES ordinals -- node @@ -2361,10 +2416,10 @@ struct clay_document { // over-invalidating costs one full walk (the rule at touch(), restated). void touch_region_structural(const math::Aabb& changed) { std::lock_guard lock(cache_mutex_); - forget_appends(); ++structure_revision_; const std::uint64_t next = revision.fetch_add(1, std::memory_order_relaxed) + 1; touch_region_locked({&changed, 1}, kFrontierDrop, next); + forget_appends(); } // The frontier invalidation (#360): a parameter edit inside root ordinal @@ -2378,9 +2433,9 @@ struct clay_document { // touch_region_from over several boxes, as touch_regions is to touch_region. void touch_regions_from(std::span changed, std::uint32_t frontier) { std::lock_guard lock(cache_mutex_); - forget_appends(); // not an append either; the log's contiguity is broken const std::uint64_t next = revision.fetch_add(1, std::memory_order_relaxed) + 1; touch_region_locked(changed, frontier, next); + forget_appends(); // not an append either; the log's contiguity is broken } std::uint64_t current_revision() const { return revision.load(std::memory_order_relaxed); } @@ -2776,6 +2831,40 @@ struct clay_document { void forget_appends() const { append_valid_ = false; append_log_.clear(); + append_reach_.clear(); + } + + // May a seed at `rev` that THIS region edit does not reach be carried + // forward to `next`? Only if nothing between `rev` and this edit reached + // it either (#665). Caller holds cache_mutex_, before forget_appends. + // + // A region invalidation advances the seeds it misses, and that is only + // sound for a seed that was current just before it: the claim is "this + // edit did not change the brick", not "nothing did". A seed can lag the + // document by more than one revision because an append re-stamps no seed + // -- the log is what carries a seed across appends, and this edit is about + // to forget the log. So a lagging seed is carried only when the log covers + // every revision it lags by and no append in that span reaches its cull + // region. Anything else stays at its old revision, which no fast path can + // serve: the next refill of that brick walks it in full. + // + // The case that found it: append an intersect and move it before a + // refill. The append changes the whole layer, the move's swept surface + // delta (#471) is far smaller, and every seed outside the sweep was + // advanced past an append that had changed it -- after which `rev == now` + // handed the pre-append field back on every refill, however often the host + // dirtied the brick. + // + // `reaches` is append_reach_ as a union tree, built once by the caller: + // one scan of the log per lagging seed was seeds x appends (see ReachTree). + bool carried_across_appends(std::uint64_t rev, const math::Aabb& cull, std::uint64_t next, + const std::optional& reaches) const { + if (rev + 1 == next) return true; + if (!reaches || !append_valid_ || append_at_ + 1 != next || rev < append_base_) + return false; + const std::size_t from = static_cast(rev - append_base_); + if (from > append_reach_.size()) return false; + return !reaches->any_from(from, cull); } // The one loop behind the three touch_region fronts; caller holds @@ -2798,6 +2887,8 @@ struct clay_document { forget_resume(); return; } + std::optional reaches; + if (append_valid_) reaches.emplace(append_reach_); for (auto it = resume_.begin(); it != resume_.end();) { const ResumeKey& k = it->first; ResumeEntry& e = it->second; @@ -2849,7 +2940,11 @@ struct clay_document { // until an accepted submit clears it in store_active. For a store // that holds no dirty entries -- every path before #360 -- this is // byte-for-byte the old behaviour. - if (e.dirty_from == kFrontierClean) e.revision = next; + // + // And only while nothing ELSE since the seed was taken reached the + // brick: see carried_across_appends (#665). + if (e.dirty_from == kFrontierClean && carried_across_appends(e.revision, cull, next, reaches)) + e.revision = next; ++it; } } @@ -2876,6 +2971,10 @@ struct clay_document { // brings the document to append_at_. mutable scene::TapeCheckpoint tape_checkpoint_; mutable std::vector append_log_; + // What each logged append could change, index for index with append_log_: + // what a region invalidation reads to decide which lagging seeds it may + // carry forward (carried_across_appends). + mutable std::vector append_reach_; mutable scene::LayerId append_layer_ = 0; mutable std::uint64_t append_base_ = 0; mutable std::uint64_t append_at_ = 0; @@ -4432,7 +4531,7 @@ clay_result apply_edit(clay_document* doc, const scene::Command& cmd, const char // and what lets the next compile reuse its prefix. Everything else, here // and at every other call site, keeps the general invalidation. if (const TailAppend appended = tail_append(doc->doc.document, cmd); appended.layer != 0) { - doc->touch_appended(appended.layer, appended.node); + doc->touch_appended(appended.layer, appended.node, reach); return CLAY_OK; } // Not an append, but its reach is known: a brick the edit cannot touch keeps diff --git a/docs/05-claycore-library.md b/docs/05-claycore-library.md index 6821caa0..89490540 100644 --- a/docs/05-claycore-library.md +++ b/docs/05-claycore-library.md @@ -618,7 +618,7 @@ The document tree the app and specs already define, owned here so every consumer - **Layers** (`voxel` | `sdf`), each with transform, visibility, resolution, material; SDF layers hold the **ordered edit list** — items apply to the combined preceding result; groups (nesting ≥ 4) carry group ops incl. None; layer instancing with shared content. An SDF layer also carries a **composition** — op, blend profile, blend radius, rounding, the same four an item carries — and the visible SDF layers FOLD under it rather than unioning (see *A layer that cuts the layers below it*). - **Influence bounds** computed per item/group (AABB ⊕ blend ⊕ rounding), used for brick dirtying, culling, and the locality guarantee (distant edits leave existing bricks bit-identical — regression-tested). - **Tape compiler**: edit list → flat tape; per-brick tape culling (only edits whose influence bound touches a brick appear in its tape — the Dreams design). -- **Cull index** (`scene/cull_index.h`): per-revision cache of everything the per-brick cull consults — item geometry bounds (splines tessellated once, not per brick), group influence bounds, the cull pad — the feathered replace's band plus the DRAG a smooth chain needs beyond it (#282): per profile `min(support, k · envelope(N))`, the envelope rising with `log2` of the layer's effective contributor count (nodes times their mirror and radial copies, each a leaf the tape folds through a seam blend) and the layer's seam `k` folded as its own term capped at what the item maxima alone would set, so the pad reaches the support only past ~800 quadratic contributors and is never wider than the pre-#335 `max(support, k)` (#335). Because it depends on the counts, the index keeps RAW per-profile maxima and resolves the envelope at read time — folding per node would freeze each node's `N` and leave an appended index below a fresh build. A **hard** blend drags nothing however its `k` reads, since its smin is a step and the running value is a plain `min()`, so it contributes nothing to that pad (#335); `Paint` and the extended modes do drag, because their colour fade and their channel both read `k`. A node's OWN bound keeps its `max(support, k)` dilation either way — in a mixed chain that is margin for the drag its smooth neighbours apply, and taking it away measured 540 → 10,105 band-clamped disagreements where narrowing the pad alone measures 540 → 540 — plus a per-batch **coarse cull plan**: one test of every chain against the union of a batch's brick regions yields survivor lists carrying the cached bounds, so each per-brick compile walks only the items near the batch instead of the whole document. Non-local items always survive; chains holding a feathered volume replace are never pruned (the feathered/hard replace choice reads what the cull dropped from that chain). Pure acceleration: the emitted tapes are byte-identical to the unindexed compile, regression-tested on an adversarial corpus (mirrors, groups, deformers, feathered volumes, non-local layers). The C ABI keys the index on the document revision, exactly as the cached whole-document tape — and, exactly as that tape, an APPEND extends the cached index rather than rebuilding it. A **brick refill resumes** on the same terms: `clay_brick_cache_eval_requests` keeps its own float32 results as per-brick seeds, and a brick carrying one evaluates the appended items onto it instead of the whole surviving edit list over every sample — **7.13 ms to 0.13 ms at 20,000 items**, bit-for-bit identical, carrying **colour** too — the seed keeps the colour the prefix reached, because a coloured walk folds a `CTapeValue` and continuing from the distance alone would fold every combine against black (8.12 ms to 0.15 ms at 20,000 items with colour requested) — and falling back in full wherever continuing would not be exact (the cull pad moved, the brick's own prefix produced no accumulator, the seed kept no colour when one is asked for, or the edit was not an append to the layer an append extends). **More than one visible SDF layer is no longer a refusal**: the seed keeps the half BENEATH the active layer as its own value — static while that layer is sculpted — and the refill applies the fold itself, through the kernel's own combine. That split is available while the SEAM it is taken at — the last visible SDF layer — folds with a plain hard Add, which is every document written before layer composition existed and every document whose top layer unions; a composed seam refuses the split and stores no seed, because the refill holds six floats and no document and cannot be told which operator to rejoin them with (`scene::layer_join_is_hard_union`). 16.41 ms to 0.17 ms at 40,000 items across two layers. `scene::compile_document_part` emits one side of that split, both sides culling under the whole document's pad, since a part compiled under a smaller pad drops items the whole compile keeps.. Bounded at 64 MB — bytes rather than bricks, since a coloured brick carries four times the floats — and evicted **least recently USED** (#346). The order is refreshed wherever a seed is used: handed out to answer a refill, rewritten by the resumed path, re-stored by the full one. Evicting by FIRST STORAGE, which is what it did, is exactly backwards for a stroke — the working set is stored at the first dab and rewritten by every dab after, so it sat nearest the front and a store under pressure dropped precisely the bricks the next dab was about to ask for, keeping ground the brush crossed once and left. Not reachable at 64 MB for a dim-8 distance-only cache (2,048 B a brick, so 32,768 of them), but a dim-16 coloured one is 64 KiB a brick, or about 1,000, which a large model does reach. The order holds ONE place per stored seed and no others: a region invalidation removes an entry's place with the entry, where it used to leave it behind — the order's own memory is not counted against the budget, so those grew outside the ceiling without bound, and a brick discarded and stored again was given a second place. The bytes counted are what the entries HOLD rather than what they are using, since a refill carrying no colour empties an entry's colour buffer without releasing it. Two carve-outs on the ceiling: the most recently used seed is kept whatever the budget says, since a budget below one brick would otherwise drop what the caller just stored, and the resumed path's rewrite is the one store that does not evict, so a stroke adding colour to entries that had none can sit above the budget until the next full refill trims it. An edit that is not an append no longer drops the lot: a seed is the value of that brick's CULLED tape, so an item whose influence misses the brick's cull region was dropped from it anyway and the seed survives, carried to the new revision. `command_influence_bound` supplies the reach, taken on BOTH sides of the apply and unioned — one side is not an answer, since an add's node is not there before and a removal's is not there after. Undoing an item no brick can reach went **20.79 ms to 2.64 ms** at 50,000 items. An edit whose reach is not known still drops everything, which stays the default. **A GESTURE STATES ITS REACH ONCE** (#358). Deriving the region per command is right for a single edit and wrong for a gesture that issues one command per node it touches: `clay_layer_move_surface` issues 257 over a 1,000-item document, and paying two `command_influence_bound` calls and a seed-store walk for each of them cost that path **1.34x** for four releases — over the device gate's tolerance, under its noise floor, so nothing failed. A drag can state its region exactly and in O(1): the warp's weight is zero outside `radius` of the centre, and a point with zero weight is not moved, so no sample outside that ball can evaluate differently. That is cheaper AND tighter than the derived union of 257 whole item bounds. `GestureRegion` invalidates once for the bracket, on the failure path too, since a gesture that applied three commands and then refused has still changed the document; a caller that cannot state its reach keeps the derived one, which is always correct. 0.134 ms → 0.097 ms, back to the pre-regression figure. **PER BRICK, not per batch** (#342). The gate used to require every brick of a call to carry a seed at one shared revision, and nothing re-stamps a seed but the refill that writes it — `touch_appended` leaves the store alone and a refill re-stamps only the bricks it filled — so a dirty window that MOVES always mixes the ground the last dab covered with ground it had not, and one disagreeing brick sent all of them down the full walk. The fast path fired only while the brush stood still. Each brick is now carried forward from its own revision, with one plan memoized per distinct revision (a moving window holds one or two) and unservable bricks falling into the miss gather alone: a four-brick window sliding one brick every third dab went **4.60 ms to 0.15 ms** per dab at 20,000 items, and its worst dab — the hitch as the brush crosses a brick plane — **19.50 ms to 1.36 ms**. A plan is also refused when the appends went to a layer OTHER than the one the suffix would extend: NodeIds are per-layer, every layer numbering from 1, so `compile_layer_suffix`'s tail check could agree by coincidence and fold the active layer's own last node on twice while the dab actually made was never evaluated (0.49 in distance, ten cells, silently). **The resumed loop no longer holds the document cache mutex while it evaluates** (#348). It used to compile AND evaluate every brick under the one lock it takes to read the seeds — serial, where the full path it replaces hands the batch to `eval_grid_batch` and spreads every row of every brick over the pool, and blocking `clay_eval_points` on another thread for the duration, since `cache_mutex_` guards the tape cache, the cull index and the append log too. The lock was held because `seed_for` hands out a raw pointer into the seed store and a concurrent refill may rewrite or evict the entry under it, so the seed is COPIED OUT under the lock instead — into the buffer the evaluation writes its answer to, which in the single-layer case is the caller's own output slot and so costs one `memcpy` and no second pass: `eval_points_seeded` reads a block's seed into its stack before it writes that block's result, so a seeded walk may run in place (#306's open question 7, answered). Compile and evaluate then run with the lock down; it is retaken to store what the bricks reached, with the revision re-checked exactly as `store_seeds` re-checks it. On a 24-thread desktop at 5,000 items, the lock beyond the cull-index copy it has to pay under it either way went **65.8 µs to 10.0 µs** for 48 bricks at a sixteen-dab suffix, **19.4 µs to 2.4 µs** at 12 bricks, and **22.4 µs to 6.8 µs** for 48 bricks at a one-dab suffix; the copy costs about **0.05 µs a brick**, inside the noise even at one brick. The deferred phase goes over `clay::parallel::ThreadPool` only when it is worth a dispatch, which is a narrower window than it looks: an empty `parallel_for` over 48 units costs 16–19 µs there, so 48 bricks at a sixteen-dab suffix is 66 µs of work and goes **66 µs serial to 31 µs pooled**, while 12 bricks of the same suffix is 21 µs and would go the wrong way, to 25. The gate is therefore a count of SAMPLE-INSTRUCTIONS — samples times appended items, so it needs no restating for another lattice or a longer suffix — set at 262,144, about three times the dispatch; below it the walk stays serial, off the lock either way. The gate is calibrated CONCURRENTLY as well, since letting several host threads refill at once is the point of coming off the lock and `ThreadPool` holds one job slot — a dispatch from a second thread replaces the advertised job, degrading toward serial rather than deadlocking. Measured at 48 bricks and a sixteen-dab suffix with 1 to 12 threads each refilling the same window, pooled stays **0.77–0.92x** the serial branch throughout, so the constant holds under concurrency; and the fan-out clay.h actually recommends — one batch of requests split across threads — puts every slice under the gate by itself, so nothing dispatches and the collision cannot arise. What is left under that lock is mostly the per-revision cull index, which the call copies to extend and which stays there, because the cache is what the lock is for. Nothing about a refill's answer depends on any of it: a new case races three refill threads and three `clay_eval_points` threads over one document and checks every window bit-for-bit against a document that never resumed, clean under `asan-ubsan` and under ThreadSanitizer — and with the copy reverted so the raw pointer stays live across the unlock, TSan reports the race the copy exists to prevent while the values still agree — the mutant's assertions pass and doctest reports SUCCESS. A sanitizer rather than a value check is therefore the only thing that can guard it, so ThreadSanitizer is a `tsan` preset and a per-pull-request CI job rather than something someone remembers to run; the whole suite is clean under it. The run needs ASLR off (`setarch -R`), or TSan aborts on an unexpected mapping before a test executes. Because the two paths are bit-identical by contract, nothing about a refill's output can say which produced it — `clay_document_resume_stats` reports cumulative `resumed_bricks` / `refilled_bricks` and the seed store's occupancy, which is what makes a fast path that stops firing visible rather than merely slow. **A seed is keyed by what it describes**, not merely by where it sits (#349): the brick coordinate, the LATTICE (dims and voxel size) and the BAND. A brick's tape is culled against `request_brick_box(req).dilated(band)`, so a narrower band drops items a wider one keeps and a seed taken under one band continued from a different field — a 0.15-band seed serving a 0.6-band request measured 9 of 512 samples wrong, worst 0.105 (two voxels), at a true distance of 0.354 that is well inside the band asked for and so a sample a submit stores rather than clamps. The cull PAD is the other term of that dilation and stays a gate rather than a key: it is a document-global maximum that moves under a single cache, where the band and the lattice are fixed properties of the cache that asked. Keying on the lattice also stops two caches over one document — a viewport and a mesher sharing a brick coordinate — from evicting each other's seeds on every call: `shaped_entry` refused the mismatched entry so the answers stayed right, but `store_seed` overwrote it and `resumed_bricks` sat at zero for both. All three caches read ONE append log, each taking its own tail of it. A stroke bumps the revision per stamp, and a rebuild walks every node recomputing bounds that did not move: **2.42 ms at 50,000 items against 0.13 ms to extend**, of which 94% was bounds. `CullIndex::append` refuses wherever it cannot be sure the document changed only by that append, so being wrong costs a rebuild rather than a wrong cull, and the index it produces is the one a rebuild gives — per-brick tapes byte-identical, held in `test_cull_index.cpp` over the adversarial corpus. **Extending it costs the dab, not the document** (#347). What was left once the rebuild was gone was an append that still walked the document twice: `CullIndex::append` re-walked the touched layer's flat node map to recompute the cull pad, and the ABI deep-copied the cached index to protect a reader that might be holding it against a plan — 0.054 ms and 0.043 ms at 20,000 items, together 79% of a 0.123 ms resumed dab. The pad is now kept as its two terms PER LAYER and raised from the appended subtree alone, including the children of a group the build never descends into: per layer a maximum of sums and a sum of maxima are the same number, which is what makes that exact rather than merely safe — one global pair of maxima would report a pad larger than a fresh build's. An append that cannot see the layer's node map grow by exactly the subtree it names refuses, so the pad can never be left BELOW what a rebuild reports, which is the direction that loses items a brick needed. And the copy is taken only when someone IS holding the index: every reader takes its handle under the cache mutex and holds it while it reads, so a use count of one under that mutex means no snapshot exists and the append extends the cached index in place. That decision is `scene::append_cached`, a free function rather than a branch inside the ABI's cache, so both halves of it are asserted directly — a stroke's consecutive appends extend one index with no copy taken, and an append made while a holder exists leaves the holder's pad and entries exactly as they were. An append at 20,000 items went **0.0542 ms → 0.000258 ms** and stopped scaling with the document — 1.00x against the same append on a document a tenth the size, where it was 11.9x, gated as a ratio in `tools/check_bench.py` — and the whole resumed dab behind it went **0.1233 ms → 0.00299 ms** with the same bricks resumed (medians of 7, load average 1.07 → 1.02 across both runs). The index carries its own append log rather than sharing the tape's, which is single-consumer by construction, and `clay_brick_cache_eval_requests` / `_device`, `clay_eval_grid`, `clay_tape_export` and the brick-mesh attribute pass all compile through it. +- **Cull index** (`scene/cull_index.h`): per-revision cache of everything the per-brick cull consults — item geometry bounds (splines tessellated once, not per brick), group influence bounds, the cull pad — the feathered replace's band plus the DRAG a smooth chain needs beyond it (#282): per profile `min(support, k · envelope(N))`, the envelope rising with `log2` of the layer's effective contributor count (nodes times their mirror and radial copies, each a leaf the tape folds through a seam blend) and the layer's seam `k` folded as its own term capped at what the item maxima alone would set, so the pad reaches the support only past ~800 quadratic contributors and is never wider than the pre-#335 `max(support, k)` (#335). Because it depends on the counts, the index keeps RAW per-profile maxima and resolves the envelope at read time — folding per node would freeze each node's `N` and leave an appended index below a fresh build. A **hard** blend drags nothing however its `k` reads, since its smin is a step and the running value is a plain `min()`, so it contributes nothing to that pad (#335); `Paint` and the extended modes do drag, because their colour fade and their channel both read `k`. A node's OWN bound keeps its `max(support, k)` dilation either way — in a mixed chain that is margin for the drag its smooth neighbours apply, and taking it away measured 540 → 10,105 band-clamped disagreements where narrowing the pad alone measures 540 → 540 — plus a per-batch **coarse cull plan**: one test of every chain against the union of a batch's brick regions yields survivor lists carrying the cached bounds, so each per-brick compile walks only the items near the batch instead of the whole document. Non-local items always survive; chains holding a feathered volume replace are never pruned (the feathered/hard replace choice reads what the cull dropped from that chain). Pure acceleration: the emitted tapes are byte-identical to the unindexed compile, regression-tested on an adversarial corpus (mirrors, groups, deformers, feathered volumes, non-local layers). The C ABI keys the index on the document revision, exactly as the cached whole-document tape — and, exactly as that tape, an APPEND extends the cached index rather than rebuilding it. A **brick refill resumes** on the same terms: `clay_brick_cache_eval_requests` keeps its own float32 results as per-brick seeds, and a brick carrying one evaluates the appended items onto it instead of the whole surviving edit list over every sample — **7.13 ms to 0.13 ms at 20,000 items**, bit-for-bit identical, carrying **colour** too — the seed keeps the colour the prefix reached, because a coloured walk folds a `CTapeValue` and continuing from the distance alone would fold every combine against black (8.12 ms to 0.15 ms at 20,000 items with colour requested) — and falling back in full wherever continuing would not be exact (the cull pad moved, the brick's own prefix produced no accumulator, the seed kept no colour when one is asked for, or the edit was not an append to the layer an append extends). **More than one visible SDF layer is no longer a refusal**: the seed keeps the half BENEATH the active layer as its own value — static while that layer is sculpted — and the refill applies the fold itself, through the kernel's own combine. That split is available while the SEAM it is taken at — the last visible SDF layer — folds with a plain hard Add, which is every document written before layer composition existed and every document whose top layer unions; a composed seam refuses the split and stores no seed, because the refill holds six floats and no document and cannot be told which operator to rejoin them with (`scene::layer_join_is_hard_union`). 16.41 ms to 0.17 ms at 40,000 items across two layers. `scene::compile_document_part` emits one side of that split, both sides culling under the whole document's pad, since a part compiled under a smaller pad drops items the whole compile keeps.. Bounded at 64 MB — bytes rather than bricks, since a coloured brick carries four times the floats — and evicted **least recently USED** (#346). The order is refreshed wherever a seed is used: handed out to answer a refill, rewritten by the resumed path, re-stored by the full one. Evicting by FIRST STORAGE, which is what it did, is exactly backwards for a stroke — the working set is stored at the first dab and rewritten by every dab after, so it sat nearest the front and a store under pressure dropped precisely the bricks the next dab was about to ask for, keeping ground the brush crossed once and left. Not reachable at 64 MB for a dim-8 distance-only cache (2,048 B a brick, so 32,768 of them), but a dim-16 coloured one is 64 KiB a brick, or about 1,000, which a large model does reach. The order holds ONE place per stored seed and no others: a region invalidation removes an entry's place with the entry, where it used to leave it behind — the order's own memory is not counted against the budget, so those grew outside the ceiling without bound, and a brick discarded and stored again was given a second place. The bytes counted are what the entries HOLD rather than what they are using, since a refill carrying no colour empties an entry's colour buffer without releasing it. Two carve-outs on the ceiling: the most recently used seed is kept whatever the budget says, since a budget below one brick would otherwise drop what the caller just stored, and the resumed path's rewrite is the one store that does not evict, so a stroke adding colour to entries that had none can sit above the budget until the next full refill trims it. An edit that is not an append no longer drops the lot: a seed is the value of that brick's CULLED tape, so an item whose influence misses the brick's cull region was dropped from it anyway and the seed survives, carried to the new revision. `command_influence_bound` supplies the reach, taken on BOTH sides of the apply and unioned — one side is not an answer, since an add's node is not there before and a removal's is not there after. Undoing an item no brick can reach went **20.79 ms to 2.64 ms** at 50,000 items. **Carried only when NOTHING since the seed was taken reached the brick** (#665). An append re-stamps no seed, since the append log is what carries one across appends, so a seed can lag by several revisions. A region invalidation forgets that log. It used to advance every seed its own box missed, including seeds still waiting to be carried across an append. Appending an intersect and moving it before a refill therefore left the un-intersected layer in the cache: the append changes the whole layer, the #471 swept move box does not cover it, and the laundered seeds then answered every later refill through the `rev == now` shortcut, however often the host dirtied the brick (1171 surface bricks against 192). The log now keeps each append's `command_influence_bound`. A lagging seed is advanced only when the log covers every revision it lags by and no append in that span reaches its cull region. A tree of range unions over the log, built once per region edit, answers that in about log(n) box tests per seed rather than a scan of the log. Anything else stays behind and is walked in full on its next refill. Seeds no append reached are still resumed. An edit whose reach is not known still drops everything, which stays the default. **A GESTURE STATES ITS REACH ONCE** (#358). Deriving the region per command is right for a single edit and wrong for a gesture that issues one command per node it touches: `clay_layer_move_surface` issues 257 over a 1,000-item document, and paying two `command_influence_bound` calls and a seed-store walk for each of them cost that path **1.34x** for four releases — over the device gate's tolerance, under its noise floor, so nothing failed. A drag can state its region exactly and in O(1): the warp's weight is zero outside `radius` of the centre, and a point with zero weight is not moved, so no sample outside that ball can evaluate differently. That is cheaper AND tighter than the derived union of 257 whole item bounds. `GestureRegion` invalidates once for the bracket, on the failure path too, since a gesture that applied three commands and then refused has still changed the document; a caller that cannot state its reach keeps the derived one, which is always correct. 0.134 ms → 0.097 ms, back to the pre-regression figure. **PER BRICK, not per batch** (#342). The gate used to require every brick of a call to carry a seed at one shared revision, and nothing re-stamps a seed but the refill that writes it — `touch_appended` leaves the store alone and a refill re-stamps only the bricks it filled — so a dirty window that MOVES always mixes the ground the last dab covered with ground it had not, and one disagreeing brick sent all of them down the full walk. The fast path fired only while the brush stood still. Each brick is now carried forward from its own revision, with one plan memoized per distinct revision (a moving window holds one or two) and unservable bricks falling into the miss gather alone: a four-brick window sliding one brick every third dab went **4.60 ms to 0.15 ms** per dab at 20,000 items, and its worst dab — the hitch as the brush crosses a brick plane — **19.50 ms to 1.36 ms**. A plan is also refused when the appends went to a layer OTHER than the one the suffix would extend: NodeIds are per-layer, every layer numbering from 1, so `compile_layer_suffix`'s tail check could agree by coincidence and fold the active layer's own last node on twice while the dab actually made was never evaluated (0.49 in distance, ten cells, silently). **The resumed loop no longer holds the document cache mutex while it evaluates** (#348). It used to compile AND evaluate every brick under the one lock it takes to read the seeds — serial, where the full path it replaces hands the batch to `eval_grid_batch` and spreads every row of every brick over the pool, and blocking `clay_eval_points` on another thread for the duration, since `cache_mutex_` guards the tape cache, the cull index and the append log too. The lock was held because `seed_for` hands out a raw pointer into the seed store and a concurrent refill may rewrite or evict the entry under it, so the seed is COPIED OUT under the lock instead — into the buffer the evaluation writes its answer to, which in the single-layer case is the caller's own output slot and so costs one `memcpy` and no second pass: `eval_points_seeded` reads a block's seed into its stack before it writes that block's result, so a seeded walk may run in place (#306's open question 7, answered). Compile and evaluate then run with the lock down; it is retaken to store what the bricks reached, with the revision re-checked exactly as `store_seeds` re-checks it. On a 24-thread desktop at 5,000 items, the lock beyond the cull-index copy it has to pay under it either way went **65.8 µs to 10.0 µs** for 48 bricks at a sixteen-dab suffix, **19.4 µs to 2.4 µs** at 12 bricks, and **22.4 µs to 6.8 µs** for 48 bricks at a one-dab suffix; the copy costs about **0.05 µs a brick**, inside the noise even at one brick. The deferred phase goes over `clay::parallel::ThreadPool` only when it is worth a dispatch, which is a narrower window than it looks: an empty `parallel_for` over 48 units costs 16–19 µs there, so 48 bricks at a sixteen-dab suffix is 66 µs of work and goes **66 µs serial to 31 µs pooled**, while 12 bricks of the same suffix is 21 µs and would go the wrong way, to 25. The gate is therefore a count of SAMPLE-INSTRUCTIONS — samples times appended items, so it needs no restating for another lattice or a longer suffix — set at 262,144, about three times the dispatch; below it the walk stays serial, off the lock either way. The gate is calibrated CONCURRENTLY as well, since letting several host threads refill at once is the point of coming off the lock and `ThreadPool` holds one job slot — a dispatch from a second thread replaces the advertised job, degrading toward serial rather than deadlocking. Measured at 48 bricks and a sixteen-dab suffix with 1 to 12 threads each refilling the same window, pooled stays **0.77–0.92x** the serial branch throughout, so the constant holds under concurrency; and the fan-out clay.h actually recommends — one batch of requests split across threads — puts every slice under the gate by itself, so nothing dispatches and the collision cannot arise. What is left under that lock is mostly the per-revision cull index, which the call copies to extend and which stays there, because the cache is what the lock is for. Nothing about a refill's answer depends on any of it: a new case races three refill threads and three `clay_eval_points` threads over one document and checks every window bit-for-bit against a document that never resumed, clean under `asan-ubsan` and under ThreadSanitizer — and with the copy reverted so the raw pointer stays live across the unlock, TSan reports the race the copy exists to prevent while the values still agree — the mutant's assertions pass and doctest reports SUCCESS. A sanitizer rather than a value check is therefore the only thing that can guard it, so ThreadSanitizer is a `tsan` preset and a per-pull-request CI job rather than something someone remembers to run; the whole suite is clean under it. The run needs ASLR off (`setarch -R`), or TSan aborts on an unexpected mapping before a test executes. Because the two paths are bit-identical by contract, nothing about a refill's output can say which produced it — `clay_document_resume_stats` reports cumulative `resumed_bricks` / `refilled_bricks` and the seed store's occupancy, which is what makes a fast path that stops firing visible rather than merely slow. **A seed is keyed by what it describes**, not merely by where it sits (#349): the brick coordinate, the LATTICE (dims and voxel size) and the BAND. A brick's tape is culled against `request_brick_box(req).dilated(band)`, so a narrower band drops items a wider one keeps and a seed taken under one band continued from a different field — a 0.15-band seed serving a 0.6-band request measured 9 of 512 samples wrong, worst 0.105 (two voxels), at a true distance of 0.354 that is well inside the band asked for and so a sample a submit stores rather than clamps. The cull PAD is the other term of that dilation and stays a gate rather than a key: it is a document-global maximum that moves under a single cache, where the band and the lattice are fixed properties of the cache that asked. Keying on the lattice also stops two caches over one document — a viewport and a mesher sharing a brick coordinate — from evicting each other's seeds on every call: `shaped_entry` refused the mismatched entry so the answers stayed right, but `store_seed` overwrote it and `resumed_bricks` sat at zero for both. All three caches read ONE append log, each taking its own tail of it. A stroke bumps the revision per stamp, and a rebuild walks every node recomputing bounds that did not move: **2.42 ms at 50,000 items against 0.13 ms to extend**, of which 94% was bounds. `CullIndex::append` refuses wherever it cannot be sure the document changed only by that append, so being wrong costs a rebuild rather than a wrong cull, and the index it produces is the one a rebuild gives — per-brick tapes byte-identical, held in `test_cull_index.cpp` over the adversarial corpus. **Extending it costs the dab, not the document** (#347). What was left once the rebuild was gone was an append that still walked the document twice: `CullIndex::append` re-walked the touched layer's flat node map to recompute the cull pad, and the ABI deep-copied the cached index to protect a reader that might be holding it against a plan — 0.054 ms and 0.043 ms at 20,000 items, together 79% of a 0.123 ms resumed dab. The pad is now kept as its two terms PER LAYER and raised from the appended subtree alone, including the children of a group the build never descends into: per layer a maximum of sums and a sum of maxima are the same number, which is what makes that exact rather than merely safe — one global pair of maxima would report a pad larger than a fresh build's. An append that cannot see the layer's node map grow by exactly the subtree it names refuses, so the pad can never be left BELOW what a rebuild reports, which is the direction that loses items a brick needed. And the copy is taken only when someone IS holding the index: every reader takes its handle under the cache mutex and holds it while it reads, so a use count of one under that mutex means no snapshot exists and the append extends the cached index in place. That decision is `scene::append_cached`, a free function rather than a branch inside the ABI's cache, so both halves of it are asserted directly — a stroke's consecutive appends extend one index with no copy taken, and an append made while a holder exists leaves the holder's pad and entries exactly as they were. An append at 20,000 items went **0.0542 ms → 0.000258 ms** and stopped scaling with the document — 1.00x against the same append on a document a tenth the size, where it was 11.9x, gated as a ratio in `tools/check_bench.py` — and the whole resumed dab behind it went **0.1233 ms → 0.00299 ms** with the same bricks resumed (medians of 7, load average 1.07 → 1.02 across both runs). The index carries its own append log rather than sharing the tape's, which is single-consumer by construction, and `clay_brick_cache_eval_requests` / `_device`, `clay_eval_grid`, `clay_tape_export` and the brick-mesh attribute pass all compile through it. - **Undo command vocabulary**: every mutation is a serializable command with an inverse (add/remove/reorder item, set-param, voxel-span edit, layer ops). The in-memory undo stack and the document file share this one vocabulary — one serialization story, tiny undo steps, stroke-level coalescing. `clay::brick`: sparse virtual grid of 8³/16³ bricks, fp16 narrow band (±3 voxels), dirty-set tracking, async-friendly (evaluation requests are plain data; the app owns threading/queues via the backend), LOD mip bricks for far view. diff --git a/openspec/changes/refill-a-dirtied-brick-against-the-current-document/.openspec.yaml b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/.openspec.yaml new file mode 100644 index 00000000..29382a2c --- /dev/null +++ b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-29 diff --git a/openspec/changes/refill-a-dirtied-brick-against-the-current-document/README.md b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/README.md new file mode 100644 index 00000000..5f313cec --- /dev/null +++ b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/README.md @@ -0,0 +1,3 @@ +# refill-a-dirtied-brick-against-the-current-document + +A region invalidation carries a brick's seed forward only when nothing since the seed was taken reached the brick (#665). diff --git a/openspec/changes/refill-a-dirtied-brick-against-the-current-document/design.md b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/design.md new file mode 100644 index 00000000..c78610e0 --- /dev/null +++ b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/design.md @@ -0,0 +1,16 @@ +## Where the reach comes from +`apply_edit` already takes `command_influence_bound` on both sides of every command before deciding it was a `TailAppend`. That box goes into `touch_appended` and is stored index for index with the log. Nothing is recomputed under `cache_mutex_`, and an intersect's box is the layer extent, which is exactly the region its append changes. + +## Why not simply drop a lagging seed +Dropping is correct, and it was measured as the mutant `return rev + 1 == next`. After a stroke most seeds on the model sit at the revision before the stroke, because a refill re-stamps only the bricks it filled. Dropping them sends every later edit there down the full walk. On the regression fixture the resumed count for a re-dirtied underside went from 108 bricks to 0 (all 144 refilled). The per-append check keeps those seeds. + +## Cost +The check runs once per untouched clean seed that lags. The first version scanned the log from each seed's revision, which made the region edit cost seeds x appends. On a filled unit sphere (dim 8, voxel 0.02) with N dabs on its cap, each refilled on its own, the first far move afterwards took: + +| dabs | main (unsound) | linear scan | range-union tree | +|---|---|---|---| +| 200 | 0.21 ms | 3.98 ms | 0.52 ms | +| 1000 | 0.21 ms | 14.5 ms | 0.89 ms | +| 3000 | 0.30 ms | 42.9 ms | 1.27 ms | + +The scan grew with the length of the stroke, under `cache_mutex_`. So the region edit now builds a binary tree of unions over the log's index ranges (`ReachTree`, O(n)) once, and each seed asks it whether any reach from its own index on meets its cull region. A node's union only prunes; the answer is decided at a leaf, so the result is exactly the scan's. Consecutive dabs lie close together, which keeps a range's union tight. An infinite reach is widened to the whole of space when it is logged, so no union can prune it. diff --git a/openspec/changes/refill-a-dirtied-brick-against-the-current-document/proposal.md b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/proposal.md new file mode 100644 index 00000000..da6503c5 --- /dev/null +++ b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/proposal.md @@ -0,0 +1,24 @@ +## Why +Issue #665: appending an `Op::Intersect` item and then moving that same item with a uniform transform, with no refill between, left the brick cache holding the un-intersected layer. Marking the item's whole influence bound dirty and refilling did not repair it. The document was right (`clay_raycast` missed outside the cutter) and the bricks were not. + +Measured on the issue's fixture (unit sphere, cylinder r 0.25 / half-height 1.6 moved to y 0.9, dim 8, voxel 0.02, band 3 voxels): + +| variant | surface bricks before | after | +|---|---|---| +| cutter built at y 0.9 in one add (reference) | 192 | 192 | +| add at origin, uniform move, refill | **1171** (the sphere) | 192, bit-identical to the reference | +| the same, refilled a second time | **1171** | 192 | +| add a local item on the sphere, move a DIFFERENT item far away, refill the layer | **1123** vs 1116 | 1116, bit-identical | + +The last row is not in the issue. It is the same defect without an intersect, and it shows that the cause is not the #471 swept bound. + +## What the issue's reading got right and wrong +The issue guessed that the append's pending state is lost when the move calls `touch_region_from` at the appended ordinal. The loss is real, but the ordinal and the frontier path are not the cause. Instrumenting `touch_region_locked` showed every stale brick's seed at revision 3 (before the append) advanced straight to revision 5 (the move). An append re-stamps no seed, because the append log is what carries a seed across appends. The region invalidation that followed forgot the log and then advanced every seed its box missed to the new revision, including seeds still waiting to be carried across the append. After that, the refill's `rev == now` shortcut returned the pre-append value on every refill, however often the host dirtied the brick. All three region fronts share this loop (`touch_regions`, `touch_region_structural`, `touch_regions_from`), so any edit kind can do it. An intersect exposes it because its append changes the whole layer, and the swept move box is intentionally much smaller than that. + +## What Changes +- The append log keeps each append's reach (the `command_influence_bound` union `apply_edit` already computes) beside the node it names. +- A region invalidation advances an unreached clean seed only if (a) it was current just before this edit, or (b) the log covers every revision the seed lags by and no append in that span reached the seed's cull region. The reaches are queried through a tree of range unions built once per region edit, so each seed costs about log(n) box tests, not n. Any other seed stays at its old revision, so the next refill of that brick walks it in full. +- All three region fronts now read the log before forgetting it. + +## Impact +No ABI, format or version change. A brick the host dirties is refilled against the current document whatever edits came before. The carry-forward that makes an edit after a stroke cheap is kept: a seed that no append reached is still advanced and still resumed. A regression test pins this with `clay_document_resume_stats`. A mutant that drops every lagging seed turns the resumed count to zero. diff --git a/openspec/changes/refill-a-dirtied-brick-against-the-current-document/specs/brick-cache/spec.md b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/specs/brick-cache/spec.md new file mode 100644 index 00000000..8fdaac58 --- /dev/null +++ b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/specs/brick-cache/spec.md @@ -0,0 +1,14 @@ +## ADDED Requirements + +### Requirement: A dirtied brick is refilled against the current document +A brick the host marks dirty SHALL be refilled to the value the CURRENT document gives it, whatever sequence of edits preceded the mark and whether or not a refill ran between them. A stored seed SHALL be carried forward across an edit only when neither that edit nor any earlier edit the seed has not yet absorbed can reach the brick's cull region. A seed that cannot be shown current SHALL NOT answer a refill as if it were. The next refill of that brick SHALL walk it in full. + +Carrying a seed forward is a performance cache and SHALL NOT be removed where it is sound: after appends that did not reach a brick, a later region edit that does not reach it either SHALL leave the brick answerable from its seed. + +#### Scenario: Append an intersect, move it, refill +- **WHEN** an intersect item is appended to a layer and then moved with a uniform transform, with no refill between, and the item's influence bound is then marked dirty and refilled +- **THEN** the cache holds exactly the bricks, bit for bit, that a document built with the item at its final place holds + +#### Scenario: A seed no append reached is still resumed +- **WHEN** a filled cache sees an append on one side of the model and a region edit elsewhere, and a region that neither reached is then re-dirtied and refilled +- **THEN** bricks of that region are answered from their seeds, and the result is bit-identical to a document built without the edit history diff --git a/openspec/changes/refill-a-dirtied-brick-against-the-current-document/tasks.md b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/tasks.md new file mode 100644 index 00000000..5d29511c --- /dev/null +++ b/openspec/changes/refill-a-dirtied-brick-against-the-current-document/tasks.md @@ -0,0 +1,10 @@ +## Implementation +- [x] Reproduce the issue's table in a C ABI test and confirm it fails on main (1171 vs 192 surface bricks). +- [x] Instrument `touch_region_locked` to confirm pre-append seeds are advanced across the append (rev 3 -> 5). +- [x] Record each append's reach in the log; carry a lagging seed only across appends that miss it. +- [x] Read the log before forgetting it in all three region fronts. +- [x] Pin that a seed no append reached is still resumed (`clay_document_resume_stats`). +- [x] Mutation-check the gate: main-like carry, drop-all, and carry-without-scan each fail at least one case. +- [x] Replace the per-seed log scan with a range-union tree (42.9 ms -> 1.27 ms for the first region edit after 3,000 dabs). +- [x] Pin the log offset across several appends: a later append that reaches a brick holds it back, and an append the brick was refilled after does not. Mutants that read the log from its start, skip one entry, or test only the first entry each fail. +- [x] Update `docs/05-claycore-library.md`. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a50041ca..3c4ad51d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -244,6 +244,7 @@ add_executable(clay_unit_tests unit/test_cull_pad_stability.cpp unit/test_node_reach_bound.cpp unit/test_intersect_delta_bound.cpp + unit/test_c_intersect_append_move.cpp unit/test_intersect_delta_oracle.cpp unit/test_c_measure_reach.cpp unit/test_worker_config.cpp diff --git a/tests/unit/test_c_intersect_append_move.cpp b/tests/unit/test_c_intersect_append_move.cpp new file mode 100644 index 00000000..de897e91 --- /dev/null +++ b/tests/unit/test_c_intersect_append_move.cpp @@ -0,0 +1,551 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "clay.h" + +// ISSUE #665: an append followed by a region edit must not launder the seeds +// the append left behind. +// +// Appending an INTERSECT item and then moving that same item with a uniform +// transform, with no refill between, left the brick cache holding the +// un-intersected layer -- and a later mark over the whole bound plus a refill +// did not repair it. The document was right; the bricks were not. +// +// The cause was in the seed store, not in the bound. An append re-stamps no +// seed (it is the one edit a seed can be carried across, by the append log). +// The move that followed took the region invalidation, which forgets the log +// and then advances every seed its box does not reach to the NEW revision -- +// including the ones still sitting at the revision BEFORE the append. For a +// local item that is harmless only because its move's box covers where it was +// appended; an intersect changes the whole layer, and the #471 swept box is +// deliberately much smaller than that. Those seeds were then "current", so the +// refill's `rev == now` shortcut handed their pre-append values straight back, +// however many times the host dirtied the brick. +// +// The table is the issue's, each variant against the same reference: the item +// built at its final place in one add, into a SEPARATE document (a second cache +// on the same document would read the same seed store and inherit the fault). + +namespace { + +constexpr int kDim = 8; +constexpr std::size_t kSamples = 8 * 8 * 8; +constexpr float kVoxel = 0.02f; +constexpr float kLift = 0.9f; + +struct Doc { + clay_document* d = nullptr; + clay_layer_id layer = 0; + Doc() { + d = clay_document_create(); + REQUIRE(d != nullptr); + REQUIRE(clay_add_sdf_layer(d, "Base", &layer) == CLAY_OK); + } + ~Doc() { clay_document_destroy(d); } + Doc(const Doc&) = delete; + Doc& operator=(const Doc&) = delete; +}; + +struct Cache { + clay_brick_cache* c = nullptr; + Cache() { + clay_brick_config cfg; + cfg.struct_size = sizeof(cfg); + REQUIRE(clay_brick_config_defaults(&cfg) == CLAY_OK); + cfg.dim = kDim; + cfg.voxel_size = kVoxel; + cfg.band_voxels = 3; + cfg.memory_budget = 0; + cfg.colors = 0; + c = clay_brick_cache_create(&cfg); + REQUIRE(c != nullptr); + } + ~Cache() { clay_brick_cache_destroy(c); } + Cache(const Cache&) = delete; + Cache& operator=(const Cache&) = delete; +}; + +clay_node_id add_item(Doc& doc, int32_t prim, const float* params, std::size_t n, int32_t op, + float y) { + clay_item* it = clay_item_create(prim, params, n); + REQUIRE(it != nullptr); + const float pos[3] = {0.0f, y, 0.0f}; + REQUIRE(clay_item_set_position(it, pos) == CLAY_OK); + REQUIRE(clay_item_set_op(it, op) == CLAY_OK); + clay_node_id id = 0; + REQUIRE(clay_layer_add_item(doc.d, doc.layer, it, &id) == CLAY_OK); + clay_item_destroy(it); + return id; +} + +clay_node_id add_sphere(Doc& doc) { + const float r = 1.0f; + return add_item(doc, CLAY_PRIM_SPHERE, &r, 1, CLAY_OP_ADD, 0.0f); +} + +// The issue's cutter: radius 0.25, half-height 1.6, intersecting. +clay_node_id add_cutter(Doc& doc, float y) { + const float params[2] = {0.25f, 1.6f}; + return add_item(doc, CLAY_PRIM_CAPPED_CYLINDER, params, 2, CLAY_OP_INTERSECT, y); +} + +void move_uniform(Doc& doc, clay_node_id node, float y) { + const float pos[3] = {0.0f, y, 0.0f}; + const float axis[3] = {0.0f, 1.0f, 0.0f}; + REQUIRE(clay_layer_set_transform(doc.d, doc.layer, node, pos, axis, 0.0f, 1.0f) == CLAY_OK); +} + +void move_nonuniform(Doc& doc, clay_node_id node, float y) { + const float pos[3] = {0.0f, y, 0.0f}; + const float axis[3] = {0.0f, 1.0f, 0.0f}; + const float scale[3] = {1.0f, 1.001f, 1.0f}; + REQUIRE(clay_layer_set_transform_nonuniform(doc.d, doc.layer, node, pos, axis, 0.0f, scale) == + CLAY_OK); +} + +// take_dirty -> eval_requests -> submit until nothing is left. +void drain(clay_brick_cache* cache, const clay_document* doc) { + constexpr std::size_t kChunk = 256; + std::vector reqs(kChunk); + std::vector values(kChunk * kSamples); + std::vector results(kChunk); + for (;;) { + std::size_t count = kChunk, remaining = 0; + REQUIRE(clay_brick_cache_take_dirty(cache, reqs.data(), &count, &remaining) == CLAY_OK); + if (count == 0) break; + REQUIRE(clay_brick_cache_eval_requests(doc, nullptr, reqs.data(), count, values.data(), + count * kSamples, nullptr, 0) == CLAY_OK); + std::size_t accepted = 0; + REQUIRE(clay_brick_cache_submit(cache, reqs.data(), count, values.data(), count * kSamples, + nullptr, 0, results.data(), &accepted) == CLAY_OK); + REQUIRE(accepted == count); + if (remaining == 0) break; + } +} + +// What the issue's host did after the edits: mark the intersect's whole +// influence bound (the layer's extent, so it covers every stale brick) dirty. +void mark_node_bound(clay_brick_cache* cache, const Doc& doc, clay_node_id node) { + float lo[3] = {0, 0, 0}, hi[3] = {0, 0, 0}; + int32_t has = 0, infinite = 0; + REQUIRE(clay_layer_node_influence_bound(doc.d, doc.layer, node, lo, hi, &has, &infinite) == + CLAY_OK); + REQUIRE(has == 1); + REQUIRE(infinite == 0); + REQUIRE(clay_brick_cache_mark_dirty(cache, lo, hi) == CLAY_OK); +} + +using Snapshot = std::map, std::vector>; + +// Every surface brick's fp16 payload, keyed by brick coordinate. +Snapshot snapshot(const clay_brick_cache* cache) { + std::size_t count = 0; + REQUIRE(clay_brick_cache_surface_bricks(cache, nullptr, &count) == CLAY_OK); + Snapshot out; + if (count == 0) return out; + std::vector keys(count * 3); + std::size_t capacity = count; + REQUIRE(clay_brick_cache_surface_bricks(cache, keys.data(), &capacity) == CLAY_OK); + std::vector halves(count * kSamples); + std::vector states(count); + REQUIRE(clay_brick_cache_read_bricks(cache, 0, keys.data(), count, 0, states.data(), + halves.data(), count * kSamples, nullptr, 0) == CLAY_OK); + for (std::size_t i = 0; i < count; ++i) { + const auto first = halves.begin() + static_cast(i * kSamples); + out[{keys[i * 3], keys[i * 3 + 1], keys[i * 3 + 2]}].assign( + first, first + static_cast(kSamples)); + } + return out; +} + +std::uint64_t surface_bricks(const clay_brick_cache* cache) { + clay_brick_stats s; + std::memset(&s, 0, sizeof s); + s.struct_size = static_cast(sizeof s); + REQUIRE(clay_brick_cache_stats(cache, &s) == CLAY_OK); + return s.surface_bricks; +} + +// The issue's probe: a ray straight down outside the cylinder, through where +// the sphere was. The intersected form has nothing there. +int32_t cache_hits_outside_cutter(const clay_brick_cache* cache) { + const float o[3] = {-0.6f, 4.0f, -0.3f}; + const float dir[3] = {0.0f, -1.0f, 0.0f}; + int32_t hit = -1; + REQUIRE(clay_brick_cache_raycast(cache, o, dir, &hit, nullptr, nullptr, nullptr) == CLAY_OK); + return hit; +} + +int32_t doc_hits_outside_cutter(const Doc& doc) { + const float o[3] = {-0.6f, 4.0f, -0.3f}; + const float dir[3] = {0.0f, -1.0f, 0.0f}; + int32_t hit = -1; + REQUIRE(clay_raycast(doc.d, o, dir, &hit, nullptr, nullptr, nullptr) == CLAY_OK); + return hit; +} + +// A document holding the sphere, with its cache fully filled. +struct Scene { + Doc doc; + Cache cache; + Scene() { + add_sphere(doc); + REQUIRE(clay_brick_cache_mark_dirty_layer(cache.c, doc.d, doc.layer) == CLAY_OK); + drain(cache.c, doc.d); + } + void finish(clay_node_id cutter) { + mark_node_bound(cache.c, doc, cutter); + drain(cache.c, doc.d); + } +}; + +// The reference: the cutter built at its place in one add, in its own document. +Snapshot reference(std::uint64_t* out_surface) { + Scene s; + s.finish(add_cutter(s.doc, kLift)); + *out_surface = surface_bricks(s.cache.c); + return snapshot(s.cache.c); +} + +void check_matches_reference(const Scene& s) { + std::uint64_t want_surface = 0; + const Snapshot want = reference(&want_surface); + // The reference itself is the intersected form, not the sphere. + REQUIRE(want_surface > 0); + REQUIRE(want_surface < 400); + CHECK(doc_hits_outside_cutter(s.doc) == 0); + CHECK(cache_hits_outside_cutter(s.cache.c) == 0); + CHECK(surface_bricks(s.cache.c) == want_surface); + const bool same_bits = snapshot(s.cache.c) == want; + CHECK(same_bits); +} + +} // namespace + +TEST_CASE("#665: append an intersect, move it uniformly, then refill -- no stale layer") { + Scene s; + const clay_node_id cutter = add_cutter(s.doc, 0.0f); + move_uniform(s.doc, cutter, kLift); // the surface-delta fast path, at the appended ordinal + s.finish(cutter); + check_matches_reference(s); +} + +TEST_CASE("#665: re-dirtying a brick after the append+move sequence still refills it") { + // The second half of the report: even after the first refill had happened, + // marking the whole bound again did not repair the cache. + Scene s; + const clay_node_id cutter = add_cutter(s.doc, 0.0f); + move_uniform(s.doc, cutter, kLift); + s.finish(cutter); + s.finish(cutter); + check_matches_reference(s); +} + +TEST_CASE("#665: append a local item, then move ANOTHER item elsewhere, then refill") { + // The same laundering without an intersect: the move's box does not cover + // where the appended item landed, so nothing but the append log could have + // carried those seeds -- and the move forgets it. + Scene s; + const float r = 0.2f; + clay_item* it = clay_item_create(CLAY_PRIM_SPHERE, &r, 1); + REQUIRE(it != nullptr); + const float far_pos[3] = {3.0f, 0.0f, 0.0f}; + REQUIRE(clay_item_set_position(it, far_pos) == CLAY_OK); + clay_node_id far_node = 0; + REQUIRE(clay_layer_add_item(s.doc.d, s.doc.layer, it, &far_node) == CLAY_OK); + clay_item_destroy(it); + REQUIRE(clay_brick_cache_mark_dirty_layer(s.cache.c, s.doc.d, s.doc.layer) == CLAY_OK); + drain(s.cache.c, s.doc.d); + + // Append a bump on the big sphere, then move the far one: its box is far + // from the bump. + const float bump_r = 0.3f; + clay_item* bump = clay_item_create(CLAY_PRIM_SPHERE, &bump_r, 1); + REQUIRE(bump != nullptr); + const float bump_pos[3] = {0.0f, 1.0f, 0.0f}; + REQUIRE(clay_item_set_position(bump, bump_pos) == CLAY_OK); + clay_node_id bump_node = 0; + REQUIRE(clay_layer_add_item(s.doc.d, s.doc.layer, bump, &bump_node) == CLAY_OK); + clay_item_destroy(bump); + const float far_axis[3] = {0.0f, 1.0f, 0.0f}; + const float far_to[3] = {3.0f, 0.1f, 0.0f}; + REQUIRE(clay_layer_set_transform(s.doc.d, s.doc.layer, far_node, far_to, far_axis, 0.0f, + 1.0f) == CLAY_OK); + REQUIRE(clay_brick_cache_mark_dirty_layer(s.cache.c, s.doc.d, s.doc.layer) == CLAY_OK); + drain(s.cache.c, s.doc.d); + + // Reference: the same final document built without the edit history. + Doc ref; + add_sphere(ref); + clay_node_id ignored = 0; + clay_item* a = clay_item_create(CLAY_PRIM_SPHERE, &r, 1); + REQUIRE(clay_item_set_position(a, far_to) == CLAY_OK); + REQUIRE(clay_layer_add_item(ref.d, ref.layer, a, &ignored) == CLAY_OK); + clay_item_destroy(a); + clay_item* b = clay_item_create(CLAY_PRIM_SPHERE, &bump_r, 1); + REQUIRE(clay_item_set_position(b, bump_pos) == CLAY_OK); + REQUIRE(clay_layer_add_item(ref.d, ref.layer, b, &ignored) == CLAY_OK); + clay_item_destroy(b); + Cache ref_cache; + REQUIRE(clay_brick_cache_mark_dirty_layer(ref_cache.c, ref.d, ref.layer) == CLAY_OK); + drain(ref_cache.c, ref.d); + CHECK(surface_bricks(s.cache.c) == surface_bricks(ref_cache.c)); + const bool same_bits = snapshot(s.cache.c) == snapshot(ref_cache.c); + CHECK(same_bits); +} + +// -- the controls: every row of the issue's table that was already right ------ + +TEST_CASE("#665 control: the cutter built at its place in one add") { + Scene s; + s.finish(add_cutter(s.doc, kLift)); + check_matches_reference(s); +} + +TEST_CASE("#665 control: append, then a NON-uniform move (no surface-delta fast path)") { + Scene s; + const clay_node_id cutter = add_cutter(s.doc, 0.0f); + move_nonuniform(s.doc, cutter, kLift); + s.finish(cutter); + CHECK(doc_hits_outside_cutter(s.doc) == 0); + CHECK(cache_hits_outside_cutter(s.cache.c) == 0); + std::uint64_t want = 0; + reference(&want); + // A 0.1% taller cylinder can touch a brick or two more; the sphere is 1171. + CHECK(surface_bricks(s.cache.c) < want + 16); +} + +TEST_CASE("#665 control: append, refill, then the uniform move") { + Scene s; + const clay_node_id cutter = add_cutter(s.doc, 0.0f); + s.finish(cutter); + move_uniform(s.doc, cutter, kLift); + s.finish(cutter); + check_matches_reference(s); +} + +TEST_CASE("#665 control: cutter at its place, refill, two uniform moves with no refill between") { + Scene s; + const clay_node_id cutter = add_cutter(s.doc, kLift); + s.finish(cutter); + move_uniform(s.doc, cutter, 0.3f); + move_uniform(s.doc, cutter, kLift); + s.finish(cutter); + check_matches_reference(s); +} + +TEST_CASE("#665 control: cutter at its place, refill, append another item, move the cutter") { + Scene s; + const clay_node_id cutter = add_cutter(s.doc, kLift); + s.finish(cutter); + // An item that the cylinder leaves nothing of, so the final form is the + // reference's: a small sphere far outside the cutter. + const float r = 0.1f; + clay_item* it = clay_item_create(CLAY_PRIM_SPHERE, &r, 1); + REQUIRE(it != nullptr); + const float pos[3] = {0.7f, 0.0f, 0.0f}; + REQUIRE(clay_item_set_position(it, pos) == CLAY_OK); + clay_node_id small = 0; + REQUIRE(clay_layer_add_item(s.doc.d, s.doc.layer, it, &small) == CLAY_OK); + clay_item_destroy(it); + move_uniform(s.doc, cutter, 0.3f); + move_uniform(s.doc, cutter, kLift); + s.finish(cutter); + CHECK(doc_hits_outside_cutter(s.doc) == 0); + CHECK(cache_hits_outside_cutter(s.cache.c) == 0); +} + +// -- and the fix must not cost the carry it narrows --------------------------- +// +// A seed an append left behind is still carried across a later region edit +// when NO append in between reached it -- which after a stroke is almost every +// seed on the model. Dropping them all would be correct and would send the +// next edit there down the full walk, brick by brick; this pins that they are +// still answered from their seeds, and still right. + +namespace { + +clay_resume_stats resume_stats(const clay_document* doc) { + clay_resume_stats s; + std::memset(&s, 0, sizeof s); + s.struct_size = static_cast(sizeof s); + REQUIRE(clay_document_resume_stats(doc, &s) == CLAY_OK); + return s; +} + +clay_node_id add_sphere_at(Doc& doc, float r, const float pos[3]) { + clay_item* it = clay_item_create(CLAY_PRIM_SPHERE, &r, 1); + REQUIRE(it != nullptr); + REQUIRE(clay_item_set_position(it, pos) == CLAY_OK); + clay_node_id id = 0; + REQUIRE(clay_layer_add_item(doc.d, doc.layer, it, &id) == CLAY_OK); + clay_item_destroy(it); + return id; +} + +} // namespace + +TEST_CASE("#665: a seed no append reached is still carried across the next region edit") { + Doc doc; + add_sphere(doc); + const float far_at[3] = {3.0f, 0.0f, 0.0f}; + const clay_node_id far_node = add_sphere_at(doc, 0.2f, far_at); + Cache cache; + REQUIRE(clay_brick_cache_mark_dirty_layer(cache.c, doc.d, doc.layer) == CLAY_OK); + drain(cache.c, doc.d); + + // A dab on the TOP of the sphere, then a move of the far item: neither + // reaches the sphere's underside. + const float bump_at[3] = {0.0f, 1.0f, 0.0f}; + add_sphere_at(doc, 0.2f, bump_at); + const float axis[3] = {0.0f, 1.0f, 0.0f}; + const float far_to[3] = {3.0f, 0.05f, 0.0f}; + REQUIRE(clay_layer_set_transform(doc.d, doc.layer, far_node, far_to, axis, 0.0f, 1.0f) == + CLAY_OK); + + // Re-dirty only the underside and refill it. + const clay_resume_stats before = resume_stats(doc.d); + const float lo[3] = {-0.3f, -1.1f, -0.3f}; + const float hi[3] = {0.3f, -0.8f, 0.3f}; + REQUIRE(clay_brick_cache_mark_dirty(cache.c, lo, hi) == CLAY_OK); + drain(cache.c, doc.d); + const clay_resume_stats after = resume_stats(doc.d); + // Not every brick of the window resumes -- one the refill proves uniform + // counts as refilled whatever its seed (the clay_resume_stats contract), + // and the same share did before this fix. Dropping the lagging seeds turns + // the resumed count to zero; that is the regression this pins. + const std::uint64_t resumed = after.resumed_bricks - before.resumed_bricks; + const std::uint64_t refilled = after.refilled_bricks - before.refilled_bricks; + CHECK(resumed > 0); + CHECK(resumed > refilled); + + // And what they were answered with is the document, bit for bit. + Doc ref; + add_sphere(ref); + add_sphere_at(ref, 0.2f, far_to); + add_sphere_at(ref, 0.2f, bump_at); + Cache ref_cache; + REQUIRE(clay_brick_cache_mark_dirty_layer(ref_cache.c, ref.d, ref.layer) == CLAY_OK); + drain(ref_cache.c, ref.d); + REQUIRE(clay_brick_cache_mark_dirty_layer(cache.c, doc.d, doc.layer) == CLAY_OK); + drain(cache.c, doc.d); + const bool same_bits = snapshot(cache.c) == snapshot(ref_cache.c); + CHECK(same_bits); +} + +// -- several appends: the carry reads the log from the seed's own revision ----- +// +// A seed refilled partway through a run of appends has absorbed the ones before +// it, and only the ones after may keep it back. Both directions are pinned: +// a later append that reaches the brick must hold it back even though an +// earlier one did not, and an earlier append that reached it must not once the +// brick was refilled after it. + +namespace { + +const float kUnderLo[3] = {-0.3f, -1.1f, -0.3f}; +const float kUnderHi[3] = {0.3f, -0.8f, 0.3f}; + +// The same final document as the history under test, built in one pass. +Snapshot reference_of(const float far_to[3], const float first_at[3], const float second_at[3]) { + Doc ref; + add_sphere(ref); + add_sphere_at(ref, 0.2f, far_to); + add_sphere_at(ref, 0.2f, first_at); + add_sphere_at(ref, 0.2f, second_at); + Cache ref_cache; + REQUIRE(clay_brick_cache_mark_dirty_layer(ref_cache.c, ref.d, ref.layer) == CLAY_OK); + drain(ref_cache.c, ref.d); + return snapshot(ref_cache.c); +} + +} // namespace + +TEST_CASE("#665: a LATER append that reaches a brick holds its seed back") { + Doc doc; + add_sphere(doc); + const float far_at[3] = {3.0f, 0.0f, 0.0f}; + const clay_node_id far_node = add_sphere_at(doc, 0.2f, far_at); + Cache cache; + REQUIRE(clay_brick_cache_mark_dirty_layer(cache.c, doc.d, doc.layer) == CLAY_OK); + drain(cache.c, doc.d); + + // First append on top (misses the underside), second on the underside, + // then a move far from both: nothing refilled in between. + const float top[3] = {0.0f, 1.0f, 0.0f}; + const float under[3] = {0.0f, -1.0f, 0.0f}; + add_sphere_at(doc, 0.2f, top); + add_sphere_at(doc, 0.2f, under); + const float axis[3] = {0.0f, 1.0f, 0.0f}; + const float far_to[3] = {3.0f, 0.05f, 0.0f}; + REQUIRE(clay_layer_set_transform(doc.d, doc.layer, far_node, far_to, axis, 0.0f, 1.0f) == + CLAY_OK); + + // The host dirties only the underside; the second append must show there. + REQUIRE(clay_brick_cache_mark_dirty(cache.c, kUnderLo, kUnderHi) == CLAY_OK); + drain(cache.c, doc.d); + const Snapshot want = reference_of(far_to, top, under); + const Snapshot got = snapshot(cache.c); + std::size_t compared = 0, differ = 0; + const float width = kDim * kVoxel; + const auto in_mark = [&](int b, int dim) { + const float lo = static_cast(b) * width; + return lo < kUnderHi[dim] && lo + width > kUnderLo[dim]; + }; + for (const auto& [key, halves] : want) { + // Only the bricks the mark overlaps: the host dirtied nothing else. + if (!in_mark(std::get<0>(key), 0) || !in_mark(std::get<1>(key), 1) || + !in_mark(std::get<2>(key), 2)) + continue; + ++compared; + const auto it = got.find(key); + if (it == got.end() || it->second != halves) ++differ; + } + REQUIRE(compared > 0); + CHECK(differ == 0); +} + +TEST_CASE("#665: an append the brick was refilled after does not hold its seed back") { + Doc doc; + add_sphere(doc); + const float far_at[3] = {3.0f, 0.0f, 0.0f}; + const clay_node_id far_node = add_sphere_at(doc, 0.2f, far_at); + Cache cache; + REQUIRE(clay_brick_cache_mark_dirty_layer(cache.c, doc.d, doc.layer) == CLAY_OK); + drain(cache.c, doc.d); + + // An append on the underside, refilled there: those seeds have absorbed it. + const float under[3] = {0.0f, -1.0f, 0.0f}; + add_sphere_at(doc, 0.2f, under); + REQUIRE(clay_brick_cache_mark_dirty(cache.c, kUnderLo, kUnderHi) == CLAY_OK); + drain(cache.c, doc.d); + // Then one on top and a far move, with no refill between. + const float top[3] = {0.0f, 1.0f, 0.0f}; + add_sphere_at(doc, 0.2f, top); + const float axis[3] = {0.0f, 1.0f, 0.0f}; + const float far_to[3] = {3.0f, 0.05f, 0.0f}; + REQUIRE(clay_layer_set_transform(doc.d, doc.layer, far_node, far_to, axis, 0.0f, 1.0f) == + CLAY_OK); + + const clay_resume_stats before = resume_stats(doc.d); + REQUIRE(clay_brick_cache_mark_dirty(cache.c, kUnderLo, kUnderHi) == CLAY_OK); + drain(cache.c, doc.d); + const clay_resume_stats after = resume_stats(doc.d); + // Reading the log from its start instead of from the seed's revision would + // count the underside append against these seeds and drop every one. + const std::uint64_t resumed = after.resumed_bricks - before.resumed_bricks; + const std::uint64_t refilled = after.refilled_bricks - before.refilled_bricks; + CHECK(resumed > 0); + CHECK(resumed > refilled); + + REQUIRE(clay_brick_cache_mark_dirty_layer(cache.c, doc.d, doc.layer) == CLAY_OK); + drain(cache.c, doc.d); + const bool same_bits = snapshot(cache.c) == reference_of(far_to, under, top); + CHECK(same_bits); +}