diff --git a/implementation/cpp/eager-graph-nil-mmap-32.hpp b/implementation/cpp/eager-graph-nil-mmap-32.hpp index 3754a4e..7dcc811 100644 --- a/implementation/cpp/eager-graph-nil-mmap-32.hpp +++ b/implementation/cpp/eager-graph-nil-mmap-32.hpp @@ -62,7 +62,9 @@ // redex that recurs can be recognized: `_memo` maps the operands of a step // to the normal form it reached, so a repeated one is a lookup. Only the // steps that recurse are memoized — the rules that answer outright are -// cheaper to redo than to remember. +// cheaper to redo than to remember — and of those, not the ones that took +// fewer than MEMO_MIN_STEPS rules, nor those of a function whose lookups +// keep missing (recall). // // Neither alone does anything: hash-consing without the memo still re-derives // every occurrence (it just writes the answers on top of each other), and the @@ -74,6 +76,25 @@ // the index it was across a collection — which is what lets every Tree a caller // is holding survive one without being registered anywhere. +// Anonymous memory, reserved rather than committed: a page is only committed +// the first time it is touched, so a region can be mapped at the most it will +// ever need and cost what is used of it. +// +// Small pages, on purpose. Huge ones (MADV_HUGEPAGE) spare the TLB, and ran +// 8-20% faster right after a run that used them; but a VM that returns freed +// memory to its host returns whole 2 MiB blocks — the ones a huge-page fault +// needs — so a run that started after a pause touched its memory at a seventh +// of the speed, and took up to twice as long. +static void *map_pages(size_t bytes) { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; +#endif + void *mem = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, flags, -1, 0); + if (mem == MAP_FAILED) throw std::runtime_error("mmap failed to reserve memory"); + return mem; +} + class EagerGraphNilMmap32 { public: using Tree = uint32_t; @@ -93,10 +114,21 @@ class EagerGraphNilMmap32 { static constexpr size_t MAX_MEMO = size_t(1) << 24; static constexpr size_t MIN_MEMO_CAP = size_t(1) << 21; + // The most slots the hash-consing table can reach: under reserve_interned's + // 0.7 load factor, 2^31 nodes take 2^32. + static constexpr size_t MAX_INTERNED = ARENA_NODES * 2; + // A step that resolved in fewer rule applications than this is cheaper to // redo than to let its entry evict a slower one from the memo (see the - // MEMOIZE pop in apply). - static constexpr uint32_t MEMO_MIN_STEPS = 16; + // MEMOIZE pop in apply). Low, because a step of a few rules can still be + // the one asked for most: a compiler's character-class test is asked of + // every character of its input, and at 16 compile_file of a 5 MB source + // took twice the steps. + static constexpr uint32_t MEMO_MIN_STEPS = 4; + + // How many lookups in a row may miss for one function before it stops being + // looked up at all (see recall). + static constexpr uint8_t MEMO_COLD = 64; struct Node { uint32_t u; @@ -120,11 +152,20 @@ class EagerGraphNilMmap32 { // this stack the collector's root set as well as the VM's. enum FrameTag : uint32_t { APPLY_TO, COMPUTE_AND_APPLY, MEMOIZE }; + // A frame is two 8-byte words, written whole and read back whole, because + // it is usually popped within a few instructions of being pushed: a load + // that the store buffer cannot answer from one earlier store (a 16-byte + // Frame read back from its four 4-byte fields, say) waits for those stores + // to retire instead — a stall on nearly every step. struct Frame { - FrameTag tag; - uint32_t arg1; - uint32_t arg2; // 0 on an APPLY_TO frame, which mark() ignores - uint32_t meta; // MEMOIZE: low bits of the step counter at push + uint64_t lo, hi; + Frame() = default; + Frame(FrameTag tag, uint32_t arg1, uint32_t arg2, uint32_t meta = 0) + : lo(tag | uint64_t(arg1) << 32), hi(arg2 | uint64_t(meta) << 32) {} + FrameTag tag() const { return FrameTag(uint32_t(lo)); } + uint32_t arg1() const { return lo >> 32; } + uint32_t arg2() const { return uint32_t(hi); } // 0 on an APPLY_TO frame, which mark() ignores + uint32_t meta() const { return hi >> 32; } // MEMOIZE: low bits of the step counter at push }; /** One memo entry: apply(a, b) normalizes to r. a == 0 marks an empty slot. */ @@ -137,6 +178,7 @@ class EagerGraphNilMmap32 { Node *_arena; uint32_t _head; // one past the highest node ever allocated Tree _free = 0; // head of the free list, chained through v + Tree _newest = ~0u; // the node alloc() made last, if no collection since size_t _live = 0; // what the last collection found, for sizing the next one size_t _budget; // collect once _head reaches this @@ -167,8 +209,15 @@ class EagerGraphNilMmap32 { // list() skips searches because of it, and deciding that two trees are equal // by comparing their indices is sound only because of it. So no node is ever // made by alloc() alone: it is made by a search that came up empty, or where - // no search could have found one (list()). - std::vector _interned; + // no search could have found one (list(), and intern() over the node made + // last). + // + // Both tables are mapped once, at the most they can reach, and re-laid in + // place: rebuild_interned() reads the arena rather than the old table, so + // nothing needs the old copy. Memory already touched costs a memset to + // re-lay; a fresh mapping costs a fault per page all over again — on a VM, a + // host round trip per page — and holds both copies at once while it grows. + Tree *const _interned = static_cast(map_pages(MAX_INTERNED * sizeof(Tree))); size_t _interned_count = 0; size_t _interned_mask = 0; @@ -176,8 +225,16 @@ class EagerGraphNilMmap32 { // replacing what was there. Bounded on purpose — this is the one table whose // natural size is the number of distinct redexes rather than the number of // live nodes, and losing an entry costs time, not correctness. - std::vector _memo; + Memo *const _memo = static_cast(map_pages(MAX_MEMO * sizeof(Memo))); size_t _memo_mask = 0; + size_t _memo_cap = 0; // the bound (see size_memo) + size_t _memo_puts = 0; // since it last grew (see memo_put) + + // Per function (the `a` of apply(a, b), by hash), how many of its lookups in + // a row have missed, up to MEMO_COLD, and past it how many it has skipped + // since one was last made (see recall). Small on purpose: it is read on every + // lookup, so it has to stay in cache where the memo cannot. + uint8_t _cold[1 << 16]; static size_t round_up_pow2(size_t n) { size_t p = MIN_TABLE; @@ -185,14 +242,25 @@ class EagerGraphNilMmap32 { return p; } - /** 64-bit finalizer: the pair of indices is the key, and it needs mixing. */ + /** + * The pair of indices is the key, and it needs mixing — but every step hashes + * before it can load anything, so the mixing is on the critical path. One + * multiply (Fibonacci hashing), with the high half, where a product mixes + * best, folded onto the low bits the tables index by. + * + * Which leaves the bits of `u` at and above a table's index width out of the + * index: bit 32 + j of the product depends on bits 0..j of u only, so in a + * table of 2^k slots (u, v) and (u + 2^k, v) always share a slot. The + * hash-consing table is sized to the live nodes, and indices outrun it only + * by what collections have freed (probes measure 2.3-2.8 slots a search); + * the memo is smaller than the arena by design, and there such pairs evict + * each other. Folding a 128-bit product instead keeps every bit, and took + * 0.15% fewer steps on the build's three heaviest tests — in 35.9 s against + * this hash's 35.4 s. + */ static uint64_t hash(uint32_t u, uint32_t v) { - uint64_t x = (uint64_t(u) << 32) | v; - x ^= x >> 33; - x *= 0xff51afd7ed558ccdULL; - x ^= x >> 33; - x *= 0xc4ceb9fe1a85ec53ULL; - return x ^ (x >> 33); + const uint64_t x = ((uint64_t(u) << 32) | v) * 0x9e3779b97f4a7c15ULL; + return x ^ (x >> 32); } /** A node, from the free list or the high-water mark. Not interned. */ @@ -205,7 +273,7 @@ class EagerGraphNilMmap32 { result = _head++; } _arena[result] = {u, v}; - return result; + return _newest = result; } /** Put a live node in the hash-consing table, which must have room for it. */ @@ -217,9 +285,13 @@ class EagerGraphNilMmap32 { ++_interned_count; } - /** Re-lay the hash-consing table at `capacity`, from the arena's live nodes. */ - void rebuild_interned(size_t capacity) { - _interned.assign(capacity, 0); + /** Re-lay the hash-consing table at `capacity`, from the arena's live nodes. + * Out of line, as is everything else apply() reaches only now and then — + * grow_memo, collect: inlined, they made apply() twice the size, and which of + * its helpers the compiler inlined then hung on any edit to it, for as much as + * 13% more instructions a step. */ + [[gnu::noinline]] void rebuild_interned(size_t capacity) { + std::fill_n(_interned, capacity, 0); _interned_mask = capacity - 1; _interned_count = 0; // A swept node is {0, next-free}: the leaf is the only live node whose left @@ -246,8 +318,22 @@ class EagerGraphNilMmap32 { } } - /** The node for this shape: the one that already exists, or a new one. */ + /** + * The node for this shape: the one that already exists, or a new one. + * + * No search when a child is the node made last: a node is made after its + * children, so nothing can have that one as a child yet (list()'s argument, + * a cell at a time). That is a quarter to a half of all the nodes a + * reduction builds — whatever it builds on what it just built — and without + * a search to wait on, the table write overlaps with the steps after it. + */ Tree intern(uint32_t u, uint32_t v) { + if (u == _newest || v == _newest) { + const Tree fresh = alloc(u, v); + insert_interned(fresh); + reserve_interned(0); + return fresh; + } const size_t i = slot(u, v); if (_interned[i]) return _interned[i]; const Tree fresh = alloc(u, v); @@ -263,9 +349,58 @@ class EagerGraphNilMmap32 { return 0; // no tree is index 0, so 0 is "miss" } + /** Record an entry, and double the memo once it has taken as many entries + * as it has slots, until it reaches its bound. */ void memo_put(uint32_t a, uint32_t b, uint32_t r) { ++stats_counters.memo_puts; _memo[hash(a, b) & _memo_mask] = {a, b, r}; + if (++_memo_puts > _memo_mask && _memo_mask + 1 < _memo_cap) grow_memo(); + } + + /** + * Double the memo in place, keeping its entries: at twice the size, an entry + * hashes to the slot it is in or to the one `half` above it, in the half being + * added — so no entry moves onto another. + */ + [[gnu::noinline]] void grow_memo() { + const size_t half = _memo_mask + 1; + std::fill_n(_memo + half, half, Memo{0, 0, 0}); + _memo_mask = 2 * half - 1; + for (Memo *m = _memo; m < _memo + half; ++m) + if (m->a && (hash(m->a, m->b) & half)) { + m[half] = *m; + *m = {0, 0, 0}; + } + _memo_puts = 0; + } + + /** + * What the memo says apply(a, b) is, or 0 — in which case the step is about + * to be reduced, and a MEMOIZE frame is pushed to record it. + * + * Unless `a` has gone cold. Most lookups are for functions that never hit: + * in compile_file of a 5 MB source, three in four are, each of them a cache + * miss into a table sized for a build, plus a write when the step is done + * that evicts something that would have. A function MEMO_COLD lookups in a + * row missed is neither looked up nor recorded, except every sixteenth of its + * own lookups, and a hit warms it again. Its own, counted in its own byte: + * with one count for all cold lookups, a function looked up between the same + * even number of others each time keeps landing on the same residue, and can + * stay unsampled for good. What is left is the lookups a cold function skips + * once it needs the memo after all — linear, not exponential: a recursion + * that calls itself twice on one argument takes about twice the steps it + * does with every lookup made. + */ + Tree recall(Tree a, Tree b) { + uint8_t &cold = _cold[uint32_t(hash(a, 0) >> 48)]; + if (cold >= MEMO_COLD && ++cold < MEMO_COLD + 16) return 0; + if (const Tree hit = memo_get(a, b)) { + cold = 0; + return hit; + } + cold = std::min(cold + 1, int(MEMO_COLD)); // a sampled miss counts afresh + _stack.emplace_back(MEMOIZE, a, b, (uint32_t)stats_counters.steps); + return 0; } /** Whether a collection's mark phase found `at` reachable. Indices 0 and 1 @@ -301,6 +436,7 @@ class EagerGraphNilMmap32 { */ void sweep() { _free = 0; + _newest = ~0u; _live = 0; // Downwards, so the list comes out in ascending order and allocation walks // the arena forwards rather than backwards. @@ -317,37 +453,37 @@ class EagerGraphNilMmap32 { } void map_arena() { - int flags = MAP_PRIVATE | MAP_ANONYMOUS; -#ifdef MAP_NORESERVE - flags |= MAP_NORESERVE; -#endif - void *mem = mmap(nullptr, ARENA_BYTES, PROT_READ | PROT_WRITE, flags, -1, 0); - if (mem == MAP_FAILED) - throw std::runtime_error("mmap failed to reserve arena"); - _arena = static_cast(mem); + _arena = static_cast(map_pages(ARENA_BYTES)); _arena[0] = {0, 0}; // index 0 reserved: 0 is the null child sentinel _arena[1] = {0, 0}; // the shared leaf _head = 2; _free = 0; + _newest = ~0u; _live = 0; } /** - * Size the memo to the budget, since it is the rest of what a collection - * bounds: a slot per eight nodes of arena, which at 12 bytes a slot is - * comparable to what the nodes cost — a long-lived module build is exactly + * Empty the memo, and bound it by the budget, since it is the rest of what a + * collection bounds: a slot per eight nodes of arena, which at 12 bytes a slot + * is comparable to what the nodes cost — a long-lived module build is exactly * the workload where remembering more reductions beats touching less memory. * Capped, because a cache stops paying for itself well before it is the size * of the thing it is caching, and direct-mapped, so the capacity is exactly * what it occupies. An unbudgeted evaluator (a one-shot benchmark run that * never calls set_budget) keeps the small cap: it exits before a large memo * pays for its footprint. + * + * The bound is reached by growing (memo_put), not laid out up front: laying + * a memo is a write to every slot — 96 MB at the default budget, which was + * most of what starting a runner cost — and a small request never fills one. */ void size_memo() { const size_t cap = _budget < ARENA_NODES ? MAX_MEMO : MIN_MEMO_CAP; - const size_t capacity = std::min(round_up_pow2(_budget / 8), cap); - _memo.assign(capacity, Memo{0, 0, 0}); - _memo_mask = capacity - 1; + _memo_cap = std::min(round_up_pow2(_budget / 8), cap); + std::fill_n(_memo, MIN_TABLE, Memo{0, 0, 0}); + _memo_mask = MIN_TABLE - 1; + _memo_puts = 0; + std::fill(std::begin(_cold), std::end(_cold), 0); } /** @@ -387,7 +523,11 @@ class EagerGraphNilMmap32 { size_memo(); } - ~EagerGraphNilMmap32() { munmap(_arena, ARENA_BYTES); } + ~EagerGraphNilMmap32() { + munmap(_arena, ARENA_BYTES); + munmap(_interned, MAX_INTERNED * sizeof(Tree)); + munmap(_memo, MAX_MEMO * sizeof(Memo)); + } EagerGraphNilMmap32(const EagerGraphNilMmap32 &) = delete; EagerGraphNilMmap32 &operator=(const EagerGraphNilMmap32 &) = delete; @@ -426,11 +566,11 @@ class EagerGraphNilMmap32 { * The memo is dropped outright: its keys are node indices too, and it is a * cache, so re-earning its entries is the cheaper correctness. */ - void collect() { + [[gnu::noinline]] void collect() { for (Tree root : _roots) mark(root); for (const Frame &f : _stack) { // a reduction in progress is live - mark(f.arg1); - mark(f.arg2); + mark(f.arg1()); + mark(f.arg2()); } // Between mark and sweep is the one moment liveness is written on the // nodes themselves, which is what lets the memo be filtered rather than @@ -439,11 +579,13 @@ class EagerGraphNilMmap32 { // nodes is about to be swept must go — the index may be reused. What the // filter keeps is reduction work; re-earning it was the old cost of every // collection. - for (Memo &m : _memo) { - if (!m.a) continue; - if (marked(m.a) && marked(m.b) && marked(m.r)) continue; - m = {0, 0, 0}; + for (Memo *m = _memo; m <= _memo + _memo_mask; ++m) { + if (!m->a) continue; + if (marked(m->a) && marked(m->b) && marked(m->r)) continue; + *m = {0, 0, 0}; } + // Swept indices will be reused by other functions. + std::fill(std::begin(_cold), std::end(_cold), 0); sweep(); ++stats_counters.gcs; stats_counters.gc_marked += _live; @@ -469,7 +611,7 @@ class EagerGraphNilMmap32 { std::string stats() { return std::to_string(allocated()) + " nodes in arena, " + std::to_string(_interned_count) + " shared, " + - std::to_string(_memo.size()) + " memo slots"; + std::to_string(_memo_mask + 1) + " memo slots"; } Tree leaf() { return 1; } @@ -557,10 +699,8 @@ class EagerGraphNilMmap32 { goto dispatch; } if (!un.v) { // apply(△(△u')y, b) = apply(apply(u', b), apply(y, b)) - const Tree hit = memo_get(a, b); - if (hit) { result = hit; goto dispatch; } - _stack.push_back({MEMOIZE, a, b}); - _stack.push_back({COMPUTE_AND_APPLY, un.u, b}); + if (const Tree hit = recall(a, b)) { result = hit; goto dispatch; } + _stack.emplace_back(COMPUTE_AND_APPLY, un.u, b); a = y; goto reduce; } @@ -573,18 +713,14 @@ class EagerGraphNilMmap32 { goto dispatch; } if (!bn.v) { // b = △d: apply(x, d) - const Tree hit = memo_get(a, b); - if (hit) { result = hit; goto dispatch; } - _stack.push_back({MEMOIZE, a, b}); + if (const Tree hit = recall(a, b)) { result = hit; goto dispatch; } a = un.v; b = bn.u; goto reduce; } { // b = △de: apply(apply(y, d), e) - const Tree hit = memo_get(a, b); - if (hit) { result = hit; goto dispatch; } - _stack.push_back({MEMOIZE, a, b}); - _stack.push_back({APPLY_TO, bn.v, 0}); + if (const Tree hit = recall(a, b)) { result = hit; goto dispatch; } + _stack.emplace_back(APPLY_TO, bn.v, 0); a = y; b = bn.u; goto reduce; @@ -597,24 +733,24 @@ class EagerGraphNilMmap32 { while (_stack.size() != base) { const Frame f = _stack.back(); _stack.pop_back(); - if (f.tag == MEMOIZE) { + if (f.tag() == MEMOIZE) { // A tail step (`b = △d`) shares its result with the step it became, // so consecutive MEMOIZE frames all record the same normal form. // Steps that resolved in a handful of rules are cheaper to redo than // to let their entries evict a slower one from the memo. - if ((uint32_t)stats_counters.steps - f.meta >= MEMO_MIN_STEPS) - memo_put(f.arg1, f.arg2, result); + if ((uint32_t)stats_counters.steps - f.meta() >= MEMO_MIN_STEPS) + memo_put(f.arg1(), f.arg2(), result); continue; } - if (f.tag == APPLY_TO) { + if (f.tag() == APPLY_TO) { // `result` is unreachable from any root for exactly as long as it // takes to become `a`, which allocates nothing. a = result; - b = f.arg1; + b = f.arg1(); } else { // COMPUTE_AND_APPLY: apply(apply(fn, arg), result) - _stack.push_back({APPLY_TO, result, 0}); - a = f.arg1; - b = f.arg2; + _stack.emplace_back(APPLY_TO, result, 0); + a = f.arg1(); + b = f.arg2(); } goto reduce; }