From a576c4e356dc58b9385e397574de4546ce400d24 Mon Sep 17 00:00:00 2001 From: visualfc Date: Tue, 21 Jul 2026 21:32:15 +0800 Subject: [PATCH] runtime/internal/runtime/maps: swiss map --- runtime/abi/helpers.go | 8 +- runtime/abi/iface.go | 41 + runtime/abi/map.go | 33 + runtime/abi/map_swiss.go | 64 + runtime/abi/type.go | 31 - runtime/internal/runtime/goarch/nowasm.go | 5 + runtime/internal/runtime/goarch/wasm.go | 5 + runtime/internal/runtime/maps/group.go | 365 +++++ runtime/internal/runtime/maps/map.go | 901 +++++++++++ runtime/internal/runtime/maps/runtime.go | 368 +++++ .../internal/runtime/maps/runtime_fast32.go | 476 ++++++ .../internal/runtime/maps/runtime_fast64.go | 516 +++++++ .../internal/runtime/maps/runtime_faststr.go | 415 ++++++ runtime/internal/runtime/maps/table.go | 1312 +++++++++++++++++ runtime/internal/runtime/maps/table_debug.go | 131 ++ runtime/internal/runtime/math/math.go | 9 +- runtime/internal/runtime/sys/intrinsics.go | 256 ++++ 17 files changed, 4899 insertions(+), 37 deletions(-) create mode 100644 runtime/abi/iface.go create mode 100644 runtime/abi/map_swiss.go create mode 100644 runtime/internal/runtime/goarch/nowasm.go create mode 100644 runtime/internal/runtime/goarch/wasm.go create mode 100644 runtime/internal/runtime/maps/group.go create mode 100644 runtime/internal/runtime/maps/map.go create mode 100644 runtime/internal/runtime/maps/runtime.go create mode 100644 runtime/internal/runtime/maps/runtime_fast32.go create mode 100644 runtime/internal/runtime/maps/runtime_fast64.go create mode 100644 runtime/internal/runtime/maps/runtime_faststr.go create mode 100644 runtime/internal/runtime/maps/table.go create mode 100644 runtime/internal/runtime/maps/table_debug.go create mode 100644 runtime/internal/runtime/sys/intrinsics.go diff --git a/runtime/abi/helpers.go b/runtime/abi/helpers.go index 85fb1a367b..406b33df08 100644 --- a/runtime/abi/helpers.go +++ b/runtime/abi/helpers.go @@ -23,11 +23,6 @@ func NoEscape(p unsafe.Pointer) unsafe.Pointer { return unsafe.Pointer(x ^ 0) } -type EmptyInterface struct { - Type *Type - Data unsafe.Pointer -} - // TypeOf returns the runtime Type of some value. func TypeOf(a any) *Type { eface := *(*EmptyInterface)(unsafe.Pointer(&a)) @@ -52,3 +47,6 @@ func EscapeToResultNonString[T any](v T) T { EscapeNonString(v) return *(*T)(NoEscape(unsafe.Pointer(&v))) } + +// ZeroValSize is the size in bytes of runtime.zeroVal. +const ZeroValSize = 1024 diff --git a/runtime/abi/iface.go b/runtime/abi/iface.go new file mode 100644 index 0000000000..f53d7e1f4f --- /dev/null +++ b/runtime/abi/iface.go @@ -0,0 +1,41 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package abi + +import "unsafe" + +// The first word of every non-empty interface type contains an *ITab. +// It records the underlying concrete type (Type), the interface type it +// is implementing (Inter), and some ancillary information. +// +// allocated in non-garbage-collected memory +type ITab struct { + Inter *InterfaceType + Type *Type + Hash uint32 // copy of Type.Hash. Used for type switches. + Fun [1]uintptr // variable sized. fun[0]==0 means Type does not implement Inter. +} + +// EmptyInterface describes the layout of a "interface{}" or a "any." +// These are represented differently than non-empty interface, as the first +// word always points to an abi.Type. +type EmptyInterface struct { + Type *Type + Data unsafe.Pointer +} + +// NonEmptyInterface describes the layout of an interface that contains any methods. +type NonEmptyInterface struct { + ITab *ITab + Data unsafe.Pointer +} + +// CommonInterface describes the layout of both [EmptyInterface] and [NonEmptyInterface]. +type CommonInterface struct { + // Either an *ITab or a *Type, unexported to avoid accidental use. + _ unsafe.Pointer + + Data unsafe.Pointer +} diff --git a/runtime/abi/map.go b/runtime/abi/map.go index e5b0a0bb6f..70a1086034 100644 --- a/runtime/abi/map.go +++ b/runtime/abi/map.go @@ -4,6 +4,8 @@ package abi +import "unsafe" + // Map constants common to several packages // runtime/runtime-gdb.py:MapTypePrinter contains its own copy const ( @@ -12,3 +14,34 @@ const ( MapMaxKeyBytes = 128 // Must fit in a uint8. MapMaxElemBytes = 128 // Must fit in a uint8. ) + +type MapType struct { + Type + Key *Type + Elem *Type + Bucket *Type // internal type representing a hash bucket + // function for hashing keys (ptr to key, seed) -> hash + Hasher func(unsafe.Pointer, uintptr) uintptr + KeySize uint8 // size of key slot + ValueSize uint8 // size of elem slot + BucketSize uint16 // size of bucket + Flags uint32 +} + +// Note: flag values must match those used in the TMAP case +// in ../cmd/compile/internal/reflectdata/reflect.go:writeType. +func (mt *MapType) IndirectKey() bool { // store ptr to key instead of key itself + return mt.Flags&1 != 0 +} +func (mt *MapType) IndirectElem() bool { // store ptr to elem instead of elem itself + return mt.Flags&2 != 0 +} +func (mt *MapType) ReflexiveKey() bool { // true if k==k for all keys + return mt.Flags&4 != 0 +} +func (mt *MapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite + return mt.Flags&8 != 0 +} +func (mt *MapType) HashMightPanic() bool { // true if hash function might panic + return mt.Flags&16 != 0 +} diff --git a/runtime/abi/map_swiss.go b/runtime/abi/map_swiss.go new file mode 100644 index 0000000000..66eaf58a62 --- /dev/null +++ b/runtime/abi/map_swiss.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package abi + +import ( + "unsafe" +) + +// Map constants common to several packages +// runtime/runtime-gdb.py:MapTypePrinter contains its own copy +const ( + // Number of bits in the group.slot count. + MapGroupSlotsBits = 3 + + // Number of slots in a group. + MapGroupSlots = 1 << MapGroupSlotsBits // 8 + + // Maximum key or elem size to keep inline (instead of mallocing per element). + // Must fit in a uint8. + // MapMaxKeyBytes = 128 + // MapMaxElemBytes = 128 + + ctrlEmpty = 0b10000000 + bitsetLSB = 0x0101010101010101 + + // Value of control word with all empty slots. + MapCtrlEmpty = bitsetLSB * uint64(ctrlEmpty) +) + +type SwissMapType struct { + Type + Key *Type + Elem *Type + Group *Type // internal type representing a slot group + // function for hashing keys (ptr to key, seed) -> hash + Hasher func(unsafe.Pointer, uintptr) uintptr + GroupSize uintptr // == Group.Size_ + SlotSize uintptr // size of key/elem slot + ElemOff uintptr // offset of elem in key/elem slot + Flags uint32 +} + +// Flag values +const ( + MapNeedKeyUpdate = 1 << iota + MapHashMightPanic + MapIndirectKey + MapIndirectElem +) + +func (mt *SwissMapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite + return mt.Flags&MapNeedKeyUpdate != 0 +} +func (mt *SwissMapType) HashMightPanic() bool { // true if hash function might panic + return mt.Flags&MapHashMightPanic != 0 +} +func (mt *SwissMapType) IndirectKey() bool { // store ptr to key instead of key itself + return mt.Flags&MapIndirectKey != 0 +} +func (mt *SwissMapType) IndirectElem() bool { // store ptr to elem instead of elem itself + return mt.Flags&MapIndirectElem != 0 +} diff --git a/runtime/abi/type.go b/runtime/abi/type.go index 4022951330..b9d6e578e7 100644 --- a/runtime/abi/type.go +++ b/runtime/abi/type.go @@ -202,37 +202,6 @@ type SliceType struct { Elem *Type // slice element type } -type MapType struct { - Type - Key *Type - Elem *Type - Bucket *Type // internal type representing a hash bucket - // function for hashing keys (ptr to key, seed) -> hash - Hasher func(unsafe.Pointer, uintptr) uintptr - KeySize uint8 // size of key slot - ValueSize uint8 // size of elem slot - BucketSize uint16 // size of bucket - Flags uint32 -} - -// Note: flag values must match those used in the TMAP case -// in ../cmd/compile/internal/reflectdata/reflect.go:writeType. -func (mt *MapType) IndirectKey() bool { // store ptr to key instead of key itself - return mt.Flags&1 != 0 -} -func (mt *MapType) IndirectElem() bool { // store ptr to elem instead of elem itself - return mt.Flags&2 != 0 -} -func (mt *MapType) ReflexiveKey() bool { // true if k==k for all keys - return mt.Flags&4 != 0 -} -func (mt *MapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite - return mt.Flags&8 != 0 -} -func (mt *MapType) HashMightPanic() bool { // true if hash function might panic - return mt.Flags&16 != 0 -} - func (t *Type) Key() *Type { if t.Kind() == Map { return (*MapType)(unsafe.Pointer(t)).Key diff --git a/runtime/internal/runtime/goarch/nowasm.go b/runtime/internal/runtime/goarch/nowasm.go new file mode 100644 index 0000000000..910948909a --- /dev/null +++ b/runtime/internal/runtime/goarch/nowasm.go @@ -0,0 +1,5 @@ +//go:build !wasm + +package goarch + +const IsWasm = 0 diff --git a/runtime/internal/runtime/goarch/wasm.go b/runtime/internal/runtime/goarch/wasm.go new file mode 100644 index 0000000000..cfe87935a0 --- /dev/null +++ b/runtime/internal/runtime/goarch/wasm.go @@ -0,0 +1,5 @@ +//go:build wasm + +package goarch + +const IsWasm = 1 diff --git a/runtime/internal/runtime/maps/group.go b/runtime/internal/runtime/maps/group.go new file mode 100644 index 0000000000..4c0aa9fead --- /dev/null +++ b/runtime/internal/runtime/maps/group.go @@ -0,0 +1,365 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" + "github.com/goplus/llgo/runtime/internal/runtime/goarch" + "github.com/goplus/llgo/runtime/internal/runtime/sys" +) + +const ( + // Maximum load factor prior to growing. + // + // 7/8 is the same load factor used by Abseil, but Abseil defaults to + // 16 slots per group, so they get two empty slots vs our one empty + // slot. We may want to reevaluate if this is best for us. + maxAvgGroupLoad = 7 + + ctrlEmpty ctrl = 0b10000000 + ctrlDeleted ctrl = 0b11111110 + + bitsetLSB = 0x0101010101010101 + bitsetMSB = 0x8080808080808080 + bitsetEmpty = bitsetLSB * uint64(ctrlEmpty) +) + +// bitset represents a set of slots within a group. +// +// The underlying representation depends on GOARCH. +// +// On AMD64, bitset uses one bit per slot, where the bit is set if the slot is +// part of the set. All of the ctrlGroup.match* methods are replaced with +// intrinsics that return this packed representation. +// +// On other architectures, bitset uses one byte per slot, where each byte is +// either 0x80 if the slot is part of the set or 0x00 otherwise. This makes it +// convenient to calculate for an entire group at once using standard +// arithmetic instructions. +type bitset uint64 + +// first returns the relative index of the first control byte in the group that +// is in the set. +// +// Preconditions: b is not 0 (empty). +func (b bitset) first() uintptr { + return bitsetFirst(b) +} + +// Portable implementation of first. +// +// On AMD64, this is replaced with an intrisic that simply does +// TrailingZeros64. There is no need to shift as the bitset is packed. +func bitsetFirst(b bitset) uintptr { + return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 +} + +// removeFirst clears the first set bit (that is, resets the least significant +// set bit to 0). +func (b bitset) removeFirst() bitset { + return b & (b - 1) +} + +// removeBelow clears all set bits below slot i (non-inclusive). +func (b bitset) removeBelow(i uintptr) bitset { + return bitsetRemoveBelow(b, i) +} + +// Portable implementation of removeBelow. +// +// On AMD64, this is replaced with an intrisic that clears the lower i bits. +func bitsetRemoveBelow(b bitset, i uintptr) bitset { + // Clear all bits below slot i's byte. + mask := (uint64(1) << (8 * uint64(i))) - 1 + return b &^ bitset(mask) +} + +// lowestSet returns true if the bit is set for the lowest index in the bitset. +// +// This is intended for use with shiftOutLowest to loop over all entries in the +// bitset regardless of whether they are set. +func (b bitset) lowestSet() bool { + return bitsetLowestSet(b) +} + +// Portable implementation of lowestSet. +// +// On AMD64, this is replaced with an intrisic that checks the lowest bit. +func bitsetLowestSet(b bitset) bool { + return b&(1<<7) != 0 +} + +// shiftOutLowest shifts the lowest entry out of the bitset. Afterwards, the +// lowest entry in the bitset corresponds to the next slot. +func (b bitset) shiftOutLowest() bitset { + return bitsetShiftOutLowest(b) +} + +// Portable implementation of shiftOutLowest. +// +// On AMD64, this is replaced with an intrisic that shifts a single bit. +func bitsetShiftOutLowest(b bitset) bitset { + return b >> 8 +} + +// count returns the number of bits set in b. +func (b bitset) count() int { + // Note: works for both bitset representations (AMD64 and generic) + return sys.OnesCount64(uint64(b)) +} + +// Each slot in the hash table has a control byte which can have one of three +// states: empty, deleted, and full. They have the following bit patterns: +// +// empty: 1 0 0 0 0 0 0 0 +// deleted: 1 1 1 1 1 1 1 0 +// full: 0 h h h h h h h // h represents the H2 hash bits +// +// TODO(prattmic): Consider inverting the top bit so that the zero value is empty. +type ctrl uint8 + +// ctrlGroup is a fixed size array of abi.MapGroupSlots control bytes +// stored in a uint64. +type ctrlGroup uint64 + +// get returns the i-th control byte. +func (g *ctrlGroup) get(i uintptr) ctrl { + if goarch.BigEndian { + return *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) + } + return *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) +} + +// set sets the i-th control byte. +func (g *ctrlGroup) set(i uintptr, c ctrl) { + if goarch.BigEndian { + *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) = c + return + } + *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) = c +} + +// setEmpty sets all the control bytes to empty. +func (g *ctrlGroup) setEmpty() { + *g = ctrlGroup(bitsetEmpty) +} + +// matchH2 returns the set of slots which are full and for which the 7-bit hash +// matches the given value. May return false positives. +func (g ctrlGroup) matchH2(h uintptr) bitset { + return ctrlGroupMatchH2(g, h) +} + +// Portable implementation of matchH2. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchH2(g ctrlGroup, h uintptr) bitset { + // NB: This generic matching routine produces false positive matches when + // h is 2^N and the control bytes have a seq of 2^N followed by 2^N+1. For + // example: if ctrls==0x0302 and h=02, we'll compute v as 0x0100. When we + // subtract off 0x0101 the first 2 bytes we'll become 0xffff and both be + // considered matches of h. The false positive matches are not a problem, + // just a rare inefficiency. Note that they only occur if there is a real + // match and never occur on ctrlEmpty, or ctrlDeleted. The subsequent key + // comparisons ensure that there is no correctness issue. + v := uint64(g) ^ (bitsetLSB * uint64(h)) + return bitset(((v - bitsetLSB) &^ v) & bitsetMSB) +} + +// matchEmpty returns the set of slots in the group that are empty. +func (g ctrlGroup) matchEmpty() bitset { + return ctrlGroupMatchEmpty(g) +} + +// Portable implementation of matchEmpty. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchEmpty(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is empty iff bit 7 is set and bit 1 is not. We could select any + // of the other bits here (e.g. v << 1 would also work). + v := uint64(g) + return bitset((v &^ (v << 6)) & bitsetMSB) +} + +// matchEmptyOrDeleted returns the set of slots in the group that are empty or +// deleted. +func (g ctrlGroup) matchEmptyOrDeleted() bitset { + return ctrlGroupMatchEmptyOrDeleted(g) +} + +// Portable implementation of matchEmptyOrDeleted. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchEmptyOrDeleted(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is empty or deleted iff bit 7 is set. + v := uint64(g) + return bitset(v & bitsetMSB) +} + +// matchFull returns the set of slots in the group that are full. +func (g ctrlGroup) matchFull() bitset { + return ctrlGroupMatchFull(g) +} + +// anyFull reports whether any slots in the group are full. +func (g ctrlGroup) anyFull() bool { + // A slot is full iff bit 7 is unset. Test whether any slot has bit 7 unset. + return (^g)&bitsetMSB != 0 +} + +// Portable implementation of matchFull. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchFull(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is full iff bit 7 is unset. + v := uint64(g) + return bitset(^v & bitsetMSB) +} + +// groupReference is a wrapper type representing a single slot group stored at +// data. +// +// A group holds abi.MapGroupSlots slots (key/elem pairs) plus their +// control word. +type groupReference struct { + // data points to the group, which is described by typ.Group and has + // layout: + // + // type group struct { + // ctrls ctrlGroup + // slots [abi.MapGroupSlots]slot + // } + // + // type slot struct { + // key typ.Key + // elem typ.Elem + // } + data unsafe.Pointer // data *typ.Group +} + +const ( + ctrlGroupsSize = unsafe.Sizeof(ctrlGroup(0)) + groupSlotsOffset = ctrlGroupsSize +) + +// alignUp rounds n up to a multiple of a. a must be a power of 2. +func alignUp(n, a uintptr) uintptr { + return (n + a - 1) &^ (a - 1) +} + +// alignUpPow2 rounds n up to the next power of 2. +// +// Returns true if round up causes overflow. +func alignUpPow2(n uint64) (uint64, bool) { + if n == 0 { + return 0, false + } + v := (uint64(1) << sys.Len64(n-1)) + if v == 0 { + return 0, true + } + return v, false +} + +// ctrls returns the group control word. +func (g *groupReference) ctrls() *ctrlGroup { + return (*ctrlGroup)(g.data) +} + +// key returns a pointer to the key at index i. +func (g *groupReference) key(typ *abi.SwissMapType, i uintptr) unsafe.Pointer { + offset := groupSlotsOffset + i*typ.SlotSize + + return unsafe.Pointer(uintptr(g.data) + offset) +} + +// elem returns a pointer to the element at index i. +func (g *groupReference) elem(typ *abi.SwissMapType, i uintptr) unsafe.Pointer { + offset := groupSlotsOffset + i*typ.SlotSize + typ.ElemOff + + return unsafe.Pointer(uintptr(g.data) + offset) +} + +// groupsReference is a wrapper type describing an array of groups stored at +// data. +type groupsReference struct { + // data points to an array of groups. See groupReference above for the + // definition of group. + data unsafe.Pointer // data *[length]typ.Group + + // lengthMask is the number of groups in data minus one (note that + // length must be a power of two). This allows computing i%length + // quickly using bitwise AND. + lengthMask uint64 +} + +// newGroups allocates a new array of length groups. +// +// Length must be a power of two. +func newGroups(typ *abi.SwissMapType, length uint64) groupsReference { + return groupsReference{ + // TODO: make the length type the same throughout. + data: newarray(typ.Group, int(length)), + lengthMask: length - 1, + } +} + +// group returns the group at index i. +func (g *groupsReference) group(typ *abi.SwissMapType, i uint64) groupReference { + // TODO(prattmic): Do something here about truncation on cast to + // uintptr on 32-bit systems? + offset := uintptr(i) * typ.GroupSize + + return groupReference{ + data: unsafe.Pointer(uintptr(g.data) + offset), + } +} + +func cloneGroup(typ *abi.SwissMapType, newGroup, oldGroup groupReference) { + typedmemmove(typ.Group, newGroup.data, oldGroup.data) + if typ.IndirectKey() { + // Deep copy keys if indirect. + for i := uintptr(0); i < abi.MapGroupSlots; i++ { + oldKey := *(*unsafe.Pointer)(oldGroup.key(typ, i)) + if oldKey == nil { + continue + } + newKey := newobject(typ.Key) + typedmemmove(typ.Key, newKey, oldKey) + *(*unsafe.Pointer)(newGroup.key(typ, i)) = newKey + } + } + if typ.IndirectElem() { + // Deep copy elems if indirect. + for i := uintptr(0); i < abi.MapGroupSlots; i++ { + oldElem := *(*unsafe.Pointer)(oldGroup.elem(typ, i)) + if oldElem == nil { + continue + } + newElem := newobject(typ.Elem) + typedmemmove(typ.Elem, newElem, oldElem) + *(*unsafe.Pointer)(newGroup.elem(typ, i)) = newElem + } + } + +} diff --git a/runtime/internal/runtime/maps/map.go b/runtime/internal/runtime/maps/map.go new file mode 100644 index 0000000000..8af322b5b3 --- /dev/null +++ b/runtime/internal/runtime/maps/map.go @@ -0,0 +1,901 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps implements Go's builtin map type. +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" + "github.com/goplus/llgo/runtime/internal/runtime/goarch" + "github.com/goplus/llgo/runtime/internal/runtime/math" + "github.com/goplus/llgo/runtime/internal/runtime/sys" +) + +// This package contains the implementation of Go's builtin map type. +// +// The map design is based on Abseil's "Swiss Table" map design +// (https://abseil.io/about/design/swisstables), with additional modifications +// to cover Go's additional requirements, discussed below. +// +// Terminology: +// - Slot: A storage location of a single key/element pair. +// - Group: A group of abi.MapGroupSlots (8) slots, plus a control word. +// - Control word: An 8-byte word which denotes whether each slot is empty, +// deleted, or used. If a slot is used, its control byte also contains the +// lower 7 bits of the hash (H2). +// - H1: Upper 57 bits of a hash. +// - H2: Lower 7 bits of a hash. +// - Table: A complete "Swiss Table" hash table. A table consists of one or +// more groups for storage plus metadata to handle operation and determining +// when to grow. +// - Map: The top-level Map type consists of zero or more tables for storage. +// The upper bits of the hash select which table a key belongs to. +// - Directory: Array of the tables used by the map. +// +// At its core, the table design is similar to a traditional open-addressed +// hash table. Storage consists of an array of groups, which effectively means +// an array of key/elem slots with some control words interspersed. Lookup uses +// the hash to determine an initial group to check. If, due to collisions, this +// group contains no match, the probe sequence selects the next group to check +// (see below for more detail about the probe sequence). +// +// The key difference occurs within a group. In a standard open-addressed +// linear probed hash table, we would check each slot one at a time to find a +// match. A swiss table utilizes the extra control word to check all 8 slots in +// parallel. +// +// Each byte in the control word corresponds to one of the slots in the group. +// In each byte, 1 bit is used to indicate whether the slot is in use, or if it +// is empty/deleted. The other 7 bits contain the lower 7 bits of the hash for +// the key in that slot. See [ctrl] for the exact encoding. +// +// During lookup, we can use some clever bitwise manipulation to compare all 8 +// 7-bit hashes against the input hash in parallel (see [ctrlGroup.matchH2]). +// That is, we effectively perform 8 steps of probing in a single operation. +// With SIMD instructions, this could be extended to 16 slots with a 16-byte +// control word. +// +// Since we only use 7 bits of the 64 bit hash, there is a 1 in 128 (~0.7%) +// probability of false positive on each slot, but that's fine: we always need +// double check each match with a standard key comparison regardless. +// +// Probing +// +// Probing is done using the upper 57 bits (H1) of the hash as an index into +// the groups array. Probing walks through the groups using quadratic probing +// until it finds a group with a match or a group with an empty slot. See +// [probeSeq] for specifics about the probe sequence. Note the probe +// invariants: the number of groups must be a power of two, and the end of a +// probe sequence must be a group with an empty slot (the table can never be +// 100% full). +// +// Deletion +// +// Probing stops when it finds a group with an empty slot. This affects +// deletion: when deleting from a completely full group, we must not mark the +// slot as empty, as there could be more slots used later in a probe sequence +// and this deletion would cause probing to stop too early. Instead, we mark +// such slots as "deleted" with a tombstone. If the group still has an empty +// slot, we don't need a tombstone and directly mark the slot empty. Insert +// prioritizes reuse of tombstones over filling an empty slots. Otherwise, +// tombstones are only completely cleared during grow, as an in-place cleanup +// complicates iteration. +// +// Growth +// +// The probe sequence depends on the number of groups. Thus, when growing the +// group count all slots must be reordered to match the new probe sequence. In +// other words, an entire table must be grown at once. +// +// In order to support incremental growth, the map splits its contents across +// multiple tables. Each table is still a full hash table, but an individual +// table may only service a subset of the hash space. Growth occurs on +// individual tables, so while an entire table must grow at once, each of these +// grows is only a small portion of a map. The maximum size of a single grow is +// limited by limiting the maximum size of a table before it is split into +// multiple tables. +// +// A map starts with a single table. Up to [maxTableCapacity], growth simply +// replaces this table with a replacement with double capacity. Beyond this +// limit, growth splits the table into two. +// +// The map uses "extendible hashing" to select which table to use. In +// extendible hashing, we use the upper bits of the hash as an index into an +// array of tables (called the "directory"). The number of bits uses increases +// as the number of tables increases. For example, when there is only 1 table, +// we use 0 bits (no selection necessary). When there are 2 tables, we use 1 +// bit to select either the 0th or 1st table. [Map.globalDepth] is the number +// of bits currently used for table selection, and by extension (1 << +// globalDepth), the size of the directory. +// +// Note that each table has its own load factor and grows independently. If the +// 1st bucket grows, it will split. We'll need 2 bits to select tables, though +// we'll have 3 tables total rather than 4. We support this by allowing +// multiple indices to point to the same table. This example: +// +// directory (globalDepth=2) +// +----+ +// | 00 | --\ +// +----+ +--> table (localDepth=1) +// | 01 | --/ +// +----+ +// | 10 | ------> table (localDepth=2) +// +----+ +// | 11 | ------> table (localDepth=2) +// +----+ +// +// Tables track the depth they were created at (localDepth). It is necessary to +// grow the directory when splitting a table where globalDepth == localDepth. +// +// Iteration +// +// Iteration is the most complex part of the map due to Go's generous iteration +// semantics. A summary of semantics from the spec: +// 1. Adding and/or deleting entries during iteration MUST NOT cause iteration +// to return the same entry more than once. +// 2. Entries added during iteration MAY be returned by iteration. +// 3. Entries modified during iteration MUST return their latest value. +// 4. Entries deleted during iteration MUST NOT be returned by iteration. +// 5. Iteration order is unspecified. In the implementation, it is explicitly +// randomized. +// +// If the map never grows, these semantics are straightforward: just iterate +// over every table in the directory and every group and slot in each table. +// These semantics all land as expected. +// +// If the map grows during iteration, things complicate significantly. First +// and foremost, we need to track which entries we already returned to satisfy +// (1). There are three types of grow: +// a. A table replaced by a single larger table. +// b. A table split into two replacement tables. +// c. Growing the directory (occurs as part of (b) if necessary). +// +// For all of these cases, the replacement table(s) will have a different probe +// sequence, so simply tracking the current group and slot indices is not +// sufficient. +// +// For (a) and (b), note that grows of tables other than the one we are +// currently iterating over are irrelevant. +// +// We handle (a) and (b) by having the iterator keep a reference to the table +// it is currently iterating over, even after the table is replaced. We keep +// iterating over the original table to maintain the iteration order and avoid +// violating (1). Any new entries added only to the replacement table(s) will +// be skipped (allowed by (2)). To avoid violating (3) or (4), while we use the +// original table to select the keys, we must look them up again in the new +// table(s) to determine if they have been modified or deleted. There is yet +// another layer of complexity if the key does not compare equal itself. See +// [Iter.Next] for the gory details. +// +// Note that for (b) once we finish iterating over the old table we'll need to +// skip the next entry in the directory, as that contains the second split of +// the old table. We can use the old table's localDepth to determine the next +// logical index to use. +// +// For (b), we must adjust the current directory index when the directory +// grows. This is more straightforward, as the directory orders remains the +// same after grow, so we just double the index if the directory size doubles. + +// Extracts the H1 portion of a hash: the 57 upper bits. +// TODO(prattmic): what about 32-bit systems? +func h1(h uintptr) uintptr { + return h >> 7 +} + +// Extracts the H2 portion of a hash: the 7 bits not used for h1. +// +// These are used as an occupied control byte. +func h2(h uintptr) uintptr { + return h & 0x7f +} + +// Note: changes here must be reflected in cmd/compile/internal/reflectdata/map.go:MapType. +type Map struct { + // The number of filled slots (i.e. the number of elements in all + // tables). Excludes deleted slots. + // Must be first (known by the compiler, for len() builtin). + used uint64 + + // seed is the hash seed, computed as a unique random number per map. + seed uintptr + + // The directory of tables. + // + // Normally dirPtr points to an array of table pointers + // + // dirPtr *[dirLen]*table + // + // The length (dirLen) of this array is `1 << globalDepth`. Multiple + // entries may point to the same table. See top-level comment for more + // details. + // + // Small map optimization: if the map always contained + // abi.MapGroupSlots or fewer entries, it fits entirely in a + // single group. In that case dirPtr points directly to a single group. + // + // dirPtr *group + // + // In this case, dirLen is 0. used counts the number of used slots in + // the group. Note that small maps never have deleted slots (as there + // is no probe sequence to maintain). + dirPtr unsafe.Pointer + dirLen int + + // The number of bits to use in table directory lookups. + globalDepth uint8 + + // The number of bits to shift out of the hash for directory lookups. + // On 64-bit systems, this is 64 - globalDepth. + globalShift uint8 + + // writing is a flag that is toggled (XOR 1) while the map is being + // written. Normally it is set to 1 when writing, but if there are + // multiple concurrent writers, then toggling increases the probability + // that both sides will detect the race. + writing uint8 + + // tombstonePossible is false if we know that no table in this map + // contains a tombstone. + tombstonePossible bool + + // clearSeq is a sequence counter of calls to Clear. It is used to + // detect map clears during iteration. + clearSeq uint64 +} + +// Use 64-bit hash on 64-bit systems, except on Wasm, where we use +// 32-bit hash (see runtime/hash32.go). +const Use64BitHash = goarch.PtrSize == 8 && goarch.IsWasm == 0 + +func depthToShift(depth uint8) uint8 { + if !Use64BitHash { + return 32 - depth + } + return 64 - depth +} + +// If m is non-nil, it should be used rather than allocating. +// +// maxAlloc should be runtime.maxAlloc. +// +// TODO(prattmic): Put maxAlloc somewhere accessible. +func NewMap(mt *abi.SwissMapType, hint uintptr, m *Map, maxAlloc uintptr) *Map { + if m == nil { + m = new(Map) + } + + m.seed = uintptr(rand()) + + if hint <= abi.MapGroupSlots { + // A small map can fill all 8 slots, so no need to increase + // target capacity. + // + // In fact, since an 8 slot group is what the first assignment + // to an empty map would allocate anyway, it doesn't matter if + // we allocate here or on the first assignment. + // + // Thus we just return without allocating. (We'll save the + // allocation completely if no assignment comes.) + + // Note that the compiler may have initialized m.dirPtr with a + // pointer to a stack-allocated group, in which case we already + // have a group. The control word is already initialized. + + return m + } + + // Full size map. + + // Set initial capacity to hold hint entries without growing in the + // average case. + targetCapacity := (hint * abi.MapGroupSlots) / maxAvgGroupLoad + if targetCapacity < hint { // overflow + return m // return an empty map. + } + + dirSize := (uint64(targetCapacity) + maxTableCapacity - 1) / maxTableCapacity + dirSize, overflow := alignUpPow2(dirSize) + if overflow || dirSize > uint64(math.MaxUintptr) { + return m // return an empty map. + } + + // Reject hints that are obviously too large. + groups, overflow := math.MulUintptr(uintptr(dirSize), maxTableCapacity) + if overflow { + return m // return an empty map. + } else { + mem, overflow := math.MulUintptr(groups, mt.GroupSize) + if overflow || mem > maxAlloc { + return m // return an empty map. + } + } + + m.globalDepth = uint8(sys.TrailingZeros64(dirSize)) + m.globalShift = depthToShift(m.globalDepth) + + directory := make([]*table, dirSize) + + for i := range directory { + // TODO: Think more about initial table capacity. + directory[i] = newTable(mt, uint64(targetCapacity)/dirSize, i, m.globalDepth) + } + + m.dirPtr = unsafe.Pointer(&directory[0]) + m.dirLen = len(directory) + + return m +} + +func NewEmptyMap() *Map { + m := new(Map) + m.seed = uintptr(rand()) + // See comment in NewMap. No need to eager allocate a group. + return m +} + +func (m *Map) directoryIndex(hash uintptr) uintptr { + if m.dirLen == 1 { + return 0 + } + return hash >> (m.globalShift & 63) +} + +func (m *Map) directoryAt(i uintptr) *table { + return *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) +} + +func (m *Map) directorySet(i uintptr, nt *table) { + *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) = nt +} + +func (m *Map) replaceTable(nt *table) { + // The number of entries that reference the same table doubles for each + // time the globalDepth grows without the table splitting. + entries := 1 << (m.globalDepth - nt.localDepth) + for i := 0; i < entries; i++ { + //m.directory[nt.index+i] = nt + m.directorySet(uintptr(nt.index+i), nt) + } +} + +func (m *Map) installTableSplit(old, left, right *table) { + if old.localDepth == m.globalDepth { + // No room for another level in the directory. Grow the + // directory. + newDir := make([]*table, m.dirLen*2) + // for i := range m.dirLen { + for i := 0; i < m.dirLen; i++ { + t := m.directoryAt(uintptr(i)) + newDir[2*i] = t + newDir[2*i+1] = t + // t may already exist in multiple indices. We should + // only update t.index once. Since the index must + // increase, seeing the original index means this must + // be the first time we've encountered this table. + if t.index == i { + t.index = 2 * i + } + } + m.globalDepth++ + m.globalShift-- + //m.directory = newDir + m.dirPtr = unsafe.Pointer(&newDir[0]) + m.dirLen = len(newDir) + } + + // N.B. left and right may still consume multiple indices if the + // directory has grown multiple times since old was last split. + left.index = old.index + m.replaceTable(left) + + entries := 1 << (m.globalDepth - left.localDepth) + right.index = left.index + entries + m.replaceTable(right) +} + +func (m *Map) Used() uint64 { + return m.used +} + +// Get performs a lookup of the key that key points to. It returns a pointer to +// the element, or false if the key doesn't exist. +func (m *Map) Get(typ *abi.SwissMapType, key unsafe.Pointer) (unsafe.Pointer, bool) { + return m.getWithoutKey(typ, key) +} + +func (m *Map) getWithKey(typ *abi.SwissMapType, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + if m.Used() == 0 { + return nil, nil, false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + return m.getWithKeySmall(typ, hash, key) + } + + idx := m.directoryIndex(hash) + return m.directoryAt(idx).getWithKey(typ, hash, key) +} + +func (m *Map) getWithoutKey(typ *abi.SwissMapType, key unsafe.Pointer) (unsafe.Pointer, bool) { + if m.Used() == 0 { + return nil, false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + return elem, ok + } + + idx := m.directoryIndex(hash) + return m.directoryAt(idx).getWithoutKey(typ, hash, key) +} + +func (m *Map) getWithKeySmall(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotKey, slotElem, true + } + + match = match.removeFirst() + } + + // No match here means key is not in the map. + // (A single group means no need to probe or check for empty). + return nil, nil, false +} + +func (m *Map) Put(typ *abi.SwissMapType, key, elem unsafe.Pointer) { + slotElem := m.PutSlot(typ, key) + typedmemmove(typ.Elem, slotElem, elem) +} + +// PutSlot returns a pointer to the element slot where an inserted element +// should be written. +// +// PutSlot never returns nil. +func (m *Map) PutSlot(typ *abi.SwissMapType, key unsafe.Pointer) unsafe.Pointer { + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmall(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + // + // TODO(prattmic): If this is an update to an existing key then + // we actually don't need to grow. + m.growToTable(typ) + } + + for { + idx := m.directoryIndex(hash) + elem, ok := m.directoryAt(idx).PutSlot(typ, m, hash, key) + if !ok { + continue + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } +} + +func (m *Map) putSlotSmall(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + return nil + } + + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +func (m *Map) growToSmall(typ *abi.SwissMapType) { + grp := newGroups(typ, 1) + m.dirPtr = grp.data + + g := groupReference{ + data: m.dirPtr, + } + g.ctrls().setEmpty() +} + +func (m *Map) growToTable(typ *abi.SwissMapType) { + tab := newTable(typ, 2*abi.MapGroupSlots, 0, 0) + + g := groupReference{ + data: m.dirPtr, + } + + for i := uintptr(0); i < abi.MapGroupSlots; i++ { + if (g.ctrls().get(i) & ctrlEmpty) == ctrlEmpty { + // Empty + continue + } + + key := g.key(typ, i) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, i) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + + tab.uncheckedPutSlot(typ, hash, key, elem) + } + + directory := make([]*table, 1) + + directory[0] = tab + + m.dirPtr = unsafe.Pointer(&directory[0]) + m.dirLen = len(directory) + + m.globalDepth = 0 + m.globalShift = depthToShift(m.globalDepth) +} + +func (m *Map) Delete(typ *abi.SwissMapType, key unsafe.Pointer) { + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return + } + + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirLen == 0 { + m.deleteSmall(typ, hash, key) + } else { + idx := m.directoryIndex(hash) + if m.directoryAt(idx).Delete(typ, m, hash, key) { + m.tombstonePossible = true + } + } + + if m.used == 0 { + // Reset the hash seed to make it more difficult for attackers + // to repeatedly trigger hash collisions. See + // https://go.dev/issue/25237. + m.seed = uintptr(rand()) + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 +} + +func (m *Map) deleteSmall(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + slotKey := g.key(typ, i) + origSlotKey := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + m.used-- + + if typ.IndirectKey() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(origSlotKey) = nil + } else if typ.Key.Pointers() { + // Only bother clearing if there are pointers. + typedmemclr(typ.Key, slotKey) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(slotElem) = nil + } else { + // Unlike keys, always clear the elem (even if + // it contains no pointers), as compound + // assignment operations depend on cleared + // deleted values. See + // https://go.dev/issue/25936. + typedmemclr(typ.Elem, slotElem) + } + + // We only have 1 group, so it is OK to immediately + // reuse deleted slots. + g.ctrls().set(i, ctrlEmpty) + return + } + match = match.removeFirst() + } +} + +// Clear deletes all entries from the map resulting in an empty map. +func (m *Map) Clear(typ *abi.SwissMapType) { + if m == nil || m.Used() == 0 && !m.tombstonePossible { + return + } + + if m.writing != 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 // toggle, see comment on writing + + if m.dirLen == 0 { + m.clearSmall(typ) + } else { + var lastTab *table + // for i := range m.dirLen { + for i := 0; i < m.dirLen; i++ { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + t.Clear(typ) + lastTab = t + } + m.used = 0 + m.tombstonePossible = false + // TODO: shrink directory? + } + m.clearSeq++ + + // Reset the hash seed to make it more difficult for attackers to + // repeatedly trigger hash collisions. See https://go.dev/issue/25237. + m.seed = uintptr(rand()) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 +} + +func (m *Map) clearSmall(typ *abi.SwissMapType) { + g := groupReference{ + data: m.dirPtr, + } + + typedmemclr(typ.Group, g.data) + g.ctrls().setEmpty() + + m.used = 0 +} + +func (m *Map) Clone(typ *abi.SwissMapType) *Map { + // Note: this should never be called with a nil map. + if m.writing != 0 { + fatal("concurrent map clone and map write") + } + + // Shallow copy the Map structure. + m2 := new(Map) + *m2 = *m + m = m2 + + // We need to just deep copy the dirPtr field. + if m.dirPtr == nil { + // delayed group allocation, nothing to do. + } else if m.dirLen == 0 { + // Clone one group. + oldGroup := groupReference{data: m.dirPtr} + newGroup := groupReference{data: newGroups(typ, 1).data} + cloneGroup(typ, newGroup, oldGroup) + m.dirPtr = newGroup.data + } else { + // Clone each (different) table. + oldDir := unsafe.Slice((**table)(m.dirPtr), m.dirLen) + newDir := make([]*table, m.dirLen) + for i, t := range oldDir { + if i > 0 && t == oldDir[i-1] { + newDir[i] = newDir[i-1] + continue + } + newDir[i] = t.clone(typ) + } + m.dirPtr = unsafe.Pointer(&newDir[0]) + } + + return m +} + +func mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error { + if !t.HashMightPanic() { + return nil + } + return mapKeyError2(t.Key, p) +} + +func mapKeyError2(t *abi.Type, p unsafe.Pointer) error { + if t.TFlag&abi.TFlagRegularMemory != 0 { + return nil + } + switch t.Kind() { + case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String: + return nil + case abi.Interface: + i := (*abi.InterfaceType)(unsafe.Pointer(t)) + var t *abi.Type + var pdata *unsafe.Pointer + if len(i.Methods) == 0 { + a := (*abi.EmptyInterface)(p) + t = a.Type + if t == nil { + return nil + } + pdata = &a.Data + } else { + a := (*abi.NonEmptyInterface)(p) + if a.ITab == nil { + return nil + } + t = a.ITab.Type + pdata = &a.Data + } + + if t.Equal == nil { + return unhashableTypeError{t} + } + + if t.IsDirectIface() { + return mapKeyError2(t, unsafe.Pointer(pdata)) + } else { + return mapKeyError2(t, *pdata) + } + case abi.Array: + a := (*abi.ArrayType)(unsafe.Pointer(t)) + for i := uintptr(0); i < a.Len; i++ { + if err := mapKeyError2(a.Elem, unsafe.Pointer(uintptr(p)+i*a.Elem.Size_)); err != nil { + return err + } + } + return nil + case abi.Struct: + s := (*abi.StructType)(unsafe.Pointer(t)) + for _, f := range s.Fields { + if f.Name_ == "_" { + continue + } + if err := mapKeyError2(f.Typ, unsafe.Pointer(uintptr(p)+f.Offset)); err != nil { + return err + } + } + return nil + default: + // Should never happen, keep this case for robustness. + return unhashableTypeError{t} + } +} + +type unhashableTypeError struct{ typ *abi.Type } + +func (unhashableTypeError) RuntimeError() {} + +func (e unhashableTypeError) Error() string { return "hash of unhashable type: " + typeString(e.typ) } + +// Pushed from runtime +// +//go:linkname typeString +func typeString(typ *abi.Type) string diff --git a/runtime/internal/runtime/maps/runtime.go b/runtime/internal/runtime/maps/runtime.go new file mode 100644 index 0000000000..2893169a3d --- /dev/null +++ b/runtime/internal/runtime/maps/runtime.go @@ -0,0 +1,368 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" +) + +// Functions below pushed from runtime. + +//go:linkname fatal +func fatal(s string) + +//go:linkname rand +func rand() uint64 + +//go:linkname typedmemmove +func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) + +//go:linkname typedmemclr +func typedmemclr(typ *abi.Type, ptr unsafe.Pointer) + +//go:linkname newarray +func newarray(typ *abi.Type, n int) unsafe.Pointer + +//go:linkname newobject +func newobject(typ *abi.Type) unsafe.Pointer + +// Pushed from runtime in order to use runtime.plainError +// +//go:linkname errNilAssign +var errNilAssign error + +// Pull from runtime. It is important that is this the exact same copy as the +// runtime because runtime.mapaccess1_fat compares the returned pointer with +// &runtime.zeroVal[0]. +// TODO: move zeroVal to internal/abi? +// +//go:linkname zeroVal runtime.zeroVal +var zeroVal [abi.ZeroValSize]byte + +// mapaccess1 returns a pointer to h[key]. Never returns nil, instead +// it will return a reference to the zero object for the elem type if +// the key is not in the map. +// NOTE: The returned pointer may keep the whole map live, so don't +// hold onto it for very long. +// +//go:linkname runtime_mapaccess1 runtime.mapaccess1 +func runtime_mapaccess1(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess1) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // race.ReadObjectPC(typ.Key, key, callerpc, pc) + // } + // if msan.Enabled && m != nil { + // msan.Read(key, typ.Key.Size_) + // } + // if asan.Enabled && m != nil { + // asan.Read(key, typ.Key.Size_) + // } + + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen <= 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + if !ok { + return unsafe.Pointer(&zeroVal[0]) + } + return elem + } + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2 runtime.mapaccess2 +func runtime_mapaccess2(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess1) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // race.ReadObjectPC(typ.Key, key, callerpc, pc) + // } + // if msan.Enabled && m != nil { + // msan.Read(key, typ.Key.Size_) + // } + // if asan.Enabled && m != nil { + // asan.Read(key, typ.Key.Size_) + // } + + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + if !ok { + return unsafe.Pointer(&zeroVal[0]), false + } + return elem, true + } + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +//go:linkname runtime_mapassign runtime.mapassign +func runtime_mapassign(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapassign) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // race.ReadObjectPC(typ.Key, key, callerpc, pc) + // } + // if msan.Enabled { + // msan.Read(key, typ.Key.Size_) + // } + // if asan.Enabled { + // asan.Read(key, typ.Key.Size_) + // } + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmall(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + + var i uintptr + + // If we found a deleted slot along the way, we + // can replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } else { + // Otherwise, use the empty slot. + i = match.first() + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + + // No empty slots in this group. Check for a deleted + // slot, which we'll use if we don't find a match later + // in the probe sequence. + // + // We only need to remember a single deleted slot. + if firstDeletedGroup.data == nil { + // Since we already checked for empty slots + // above, matches here must be deleted slots. + match = g.ctrls().matchEmptyOrDeleted() + if match != 0 { + firstDeletedGroup = g + firstDeletedSlot = match.first() + } + } + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} diff --git a/runtime/internal/runtime/maps/runtime_fast32.go b/runtime/internal/runtime/maps/runtime_fast32.go new file mode 100644 index 0000000000..66c3d1345b --- /dev/null +++ b/runtime/internal/runtime/maps/runtime_fast32.go @@ -0,0 +1,476 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" +) + +//go:linkname runtime_mapaccess1_fast32 runtime.mapaccess1_fast32 +func runtime_mapaccess1_fast32(typ *abi.SwissMapType, m *Map, key uint32) unsafe.Pointer { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess1_fast32) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint32)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]) + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_fast32 runtime.mapaccess2_fast32 +func runtime_mapaccess2_fast32(typ *abi.SwissMapType, m *Map, key uint32) (unsafe.Pointer, bool) { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess2_fast32) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint32)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem, true + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]), false + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFast32(typ *abi.SwissMapType, hash uintptr, key uint32) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*uint32)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_fast32 runtime.mapassign_fast32 +func runtime_mapassign_fast32(typ *abi.SwissMapType, m *Map, key uint32) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapassign_fast32) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFast32(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*uint32)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +// Key is a 32-bit pointer (only called on 32-bit GOARCH). This source is identical to fast64ptr. +// +// TODO(prattmic): With some compiler refactoring we could avoid duplication of this function. +// +//go:linkname runtime_mapassign_fast32ptr runtime.mapassign_fast32ptr +func runtime_mapassign_fast32ptr(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapassign_fast32ptr) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFastPtr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot we + // find, which we'll use to insert the new entry if necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_fast32 runtime.mapdelete_fast32 +func runtime_mapdelete_fast32(typ *abi.SwissMapType, m *Map, key uint32) { + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapdelete_fast32) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/runtime/internal/runtime/maps/runtime_fast64.go b/runtime/internal/runtime/maps/runtime_fast64.go new file mode 100644 index 0000000000..c206110ca6 --- /dev/null +++ b/runtime/internal/runtime/maps/runtime_fast64.go @@ -0,0 +1,516 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" +) + +//go:linkname runtime_mapaccess1_fast64 runtime.mapaccess1_fast64 +func runtime_mapaccess1_fast64(typ *abi.SwissMapType, m *Map, key uint64) unsafe.Pointer { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess1_fast64) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint64)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]) + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_fast64 runtime.mapaccess2_fast64 +func runtime_mapaccess2_fast64(typ *abi.SwissMapType, m *Map, key uint64) (unsafe.Pointer, bool) { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess2_fast64) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint64)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem, true + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]), false + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFast64(typ *abi.SwissMapType, hash uintptr, key uint64) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*uint64)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_fast64 runtime.mapassign_fast64 +func runtime_mapassign_fast64(typ *abi.SwissMapType, m *Map, key uint64) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapassign_fast64) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFast64(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*uint64)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +func (m *Map) putSlotSmallFastPtr(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +// Key is a 64-bit pointer (only called on 64-bit GOARCH). +// +//go:linkname runtime_mapassign_fast64ptr runtime.mapassign_fast64ptr +func runtime_mapassign_fast64ptr(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapassign_fast64ptr) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFastPtr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_fast64 runtime.mapdelete_fast64 +func runtime_mapdelete_fast64(typ *abi.SwissMapType, m *Map, key uint64) { + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapdelete_fast64) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/runtime/internal/runtime/maps/runtime_faststr.go b/runtime/internal/runtime/maps/runtime_faststr.go new file mode 100644 index 0000000000..4ffd6f2152 --- /dev/null +++ b/runtime/internal/runtime/maps/runtime_faststr.go @@ -0,0 +1,415 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" + "github.com/goplus/llgo/runtime/internal/runtime/goarch" +) + +func (m *Map) getWithoutKeySmallFastStr(typ *abi.SwissMapType, key string) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + ctrls := *g.ctrls() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + + // The 64 threshold was chosen based on performance of BenchmarkMapStringKeysEight, + // where there are 8 keys to check, all of which don't quick-match the lookup key. + // In that case, we can save hashing the lookup key. That savings is worth this extra code + // for strings that are long enough that hashing is expensive. + if len(key) > 64 { + // String hashing and equality might be expensive. Do a quick check first. + j := abi.MapGroupSlots + // for i := range abi.MapGroupSlots { + for i := 0; i < abi.MapGroupSlots; i++ { + if ctrls&(1<<7) == 0 && longStringQuickEqualityTest(key, *(*string)(slotKey)) { + if j < abi.MapGroupSlots { + // 2 strings both passed the quick equality test. + // Break out of this loop and do it the slow way. + goto dohash + } + j = i + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + ctrls >>= 8 + } + if j == abi.MapGroupSlots { + // No slot passed the quick test. + return nil + } + // There's exactly one slot that passed the quick test. Do the single expensive comparison. + slotKey = g.key(typ, uintptr(j)) + if key == *(*string)(slotKey) { + return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + } + return nil + } + +dohash: + // This path will cost 1 hash and 1+ε comparisons. + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&key)), m.seed) + h2 := uint8(h2(hash)) + ctrls = *g.ctrls() + slotKey = g.key(typ, 0) + + // for range abi.MapGroupSlots { + for i := 0; i < abi.MapGroupSlots; i++ { + if uint8(ctrls) == h2 && key == *(*string)(slotKey) { + return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + ctrls >>= 8 + } + return nil +} + +// Returns true if a and b might be equal. +// Returns false if a and b are definitely not equal. +// Requires len(a)>=8. +func longStringQuickEqualityTest(a, b string) bool { + if len(a) != len(b) { + return false + } + x, y := stringPtr(a), stringPtr(b) + // Check first 8 bytes. + if *(*[8]byte)(x) != *(*[8]byte)(y) { + return false + } + // Check last 8 bytes. + x = unsafe.Pointer(uintptr(x) + uintptr(len(a)) - 8) + y = unsafe.Pointer(uintptr(y) + uintptr(len(a)) - 8) + if *(*[8]byte)(x) != *(*[8]byte)(y) { + return false + } + return true +} +func stringPtr(s string) unsafe.Pointer { + type stringStruct struct { + ptr unsafe.Pointer + len int + } + return (*stringStruct)(unsafe.Pointer(&s)).ptr +} + +//go:linkname runtime_mapaccess1_faststr runtime.mapaccess1_faststr +func runtime_mapaccess1_faststr(typ *abi.SwissMapType, m *Map, key string) unsafe.Pointer { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess1_faststr) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen <= 0 { + elem := m.getWithoutKeySmallFastStr(typ, key) + if elem == nil { + return unsafe.Pointer(&zeroVal[0]) + } + return elem + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_faststr runtime.mapaccess2_faststr +func runtime_mapaccess2_faststr(typ *abi.SwissMapType, m *Map, key string) (unsafe.Pointer, bool) { + // if race.Enabled && m != nil { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapaccess2_faststr) + // race.ReadPC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen <= 0 { + elem := m.getWithoutKeySmallFastStr(typ, key) + if elem == nil { + return unsafe.Pointer(&zeroVal[0]), false + } + return elem, true + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFastStr(typ *abi.SwissMapType, hash uintptr, key string) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + // Key needs update, as the backing storage may differ. + *(*string)(slotKey) = key + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*string)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_faststr runtime.mapassign_faststr +func runtime_mapassign_faststr(typ *abi.SwissMapType, m *Map, key string) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapassign_faststr) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFastStr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + // Key needs update, as the backing + // storage may differ. + *(*string)(slotKey) = key + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*string)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_faststr runtime.mapdelete_faststr +func runtime_mapdelete_faststr(typ *abi.SwissMapType, m *Map, key string) { + // if race.Enabled { + // callerpc := sys.GetCallerPC() + // pc := abi.FuncPCABIInternal(runtime_mapdelete_faststr) + // race.WritePC(unsafe.Pointer(m), callerpc, pc) + // } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/runtime/internal/runtime/maps/table.go b/runtime/internal/runtime/maps/table.go new file mode 100644 index 0000000000..62225f1216 --- /dev/null +++ b/runtime/internal/runtime/maps/table.go @@ -0,0 +1,1312 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" + "github.com/goplus/llgo/runtime/internal/runtime/math" +) + +// Maximum size of a table before it is split at the directory level. +// +// TODO: Completely made up value. This should be tuned for performance vs grow +// latency. +// TODO: This should likely be based on byte size, as copying costs will +// dominate grow latency for large objects. +const maxTableCapacity = 1024 + +// Ensure the max capacity fits in uint16, used for capacity and growthLeft +// below. +var _ = uint16(maxTableCapacity) + +// table is a Swiss table hash table structure. +// +// Each table is a complete hash table implementation. +// +// Map uses one or more tables to store entries. Extendible hashing (hash +// prefix) is used to select the table to use for a specific key. Using +// multiple tables enables incremental growth by growing only one table at a +// time. +type table struct { + // The number of filled slots (i.e. the number of elements in the table). + used uint16 + + // The total number of slots (always 2^N). Equal to + // `(groups.lengthMask+1)*abi.MapGroupSlots`. + capacity uint16 + + // The number of slots we can still fill without needing to rehash. + // + // We rehash when used + tombstones > loadFactor*capacity, including + // tombstones so the table doesn't overfill with tombstones. This field + // counts down remaining empty slots before the next rehash. + growthLeft uint16 + + // The number of bits used by directory lookups above this table. Note + // that this may be less then globalDepth, if the directory has grown + // but this table has not yet been split. + localDepth uint8 + + // Index of this table in the Map directory. This is the index of the + // _first_ location in the directory. The table may occur in multiple + // sequential indices. + // + // index is -1 if the table is stale (no longer installed in the + // directory). + index int + + // groups is an array of slot groups. Each group holds abi.MapGroupSlots + // key/elem slots and their control bytes. A table has a fixed size + // groups array. The table is replaced (in rehash) when more space is + // required. + // + // TODO(prattmic): keys and elements are interleaved to maximize + // locality, but it comes at the expense of wasted space for some types + // (consider uint8 key, uint64 element). Consider placing all keys + // together in these cases to save space. + groups groupsReference +} + +func newTable(typ *abi.SwissMapType, capacity uint64, index int, localDepth uint8) *table { + if capacity < abi.MapGroupSlots { + capacity = abi.MapGroupSlots + } + + t := &table{ + index: index, + localDepth: localDepth, + } + + if capacity > maxTableCapacity { + panic("initial table capacity too large") + } + + // N.B. group count must be a power of two for probeSeq to visit every + // group. + capacity, overflow := alignUpPow2(capacity) + if overflow { + panic("rounded-up capacity overflows uint64") + } + + t.reset(typ, uint16(capacity)) + + return t +} + +// reset resets the table with new, empty groups with the specified new total +// capacity. +func (t *table) reset(typ *abi.SwissMapType, capacity uint16) { + groupCount := uint64(capacity) / abi.MapGroupSlots + t.groups = newGroups(typ, groupCount) + t.capacity = capacity + t.growthLeft = t.maxGrowthLeft() + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + g.ctrls().setEmpty() + } +} + +// maxGrowthLeft is the number of inserts we can do before +// resizing, starting from an empty table. +func (t *table) maxGrowthLeft() uint16 { + if t.capacity == 0 { + // No real reason to support zero capacity table, since an + // empty Map simply won't have a table. + panic("table must have positive capacity") + } else if t.capacity <= abi.MapGroupSlots { + // If the map fits in a single group then we're able to fill all of + // the slots except 1 (an empty slot is needed to terminate find + // operations). + // + // TODO(go.dev/issue/54766): With a special case in probing for + // single-group tables, we could fill all slots. + return t.capacity - 1 + } else { + if t.capacity > math.MaxUint16/maxAvgGroupLoad { + panic("overflow") + } + return (t.capacity * maxAvgGroupLoad) / abi.MapGroupSlots + } + +} + +func (t *table) Used() uint64 { + return uint64(t.used) +} + +// Get performs a lookup of the key that key points to. It returns a pointer to +// the element, or false if the key doesn't exist. +func (t *table) Get(typ *abi.SwissMapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { + // TODO(prattmic): We could avoid hashing in a variety of special + // cases. + // + // - One entry maps could just directly compare the single entry + // without hashing. + // - String keys could do quick checks of a few bytes before hashing. + hash := typ.Hasher(key, m.seed) + _, elem, ok := t.getWithKey(typ, hash, key) + return elem, ok +} + +// getWithKey performs a lookup of key, returning a pointer to the version of +// the key in the map in addition to the element. +// +// This is relevant when multiple different key values compare equal (e.g., +// +0.0 and -0.0). When a grow occurs during iteration, iteration perform a +// lookup of keys from the old group in the new group in order to correctly +// expose updated elements. For NeedsKeyUpdate keys, iteration also must return +// the new key value, not the old key value. +// hash must be the hash of the key. +func (t *table) getWithKey(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + // To find the location of a key in the table, we compute hash(key). From + // h1(hash(key)) and the capacity, we construct a probeSeq that visits + // every group of slots in some interesting order. See [probeSeq]. + // + // We walk through these indices. At each index, we select the entire + // group starting with that index and extract potential candidates: + // occupied slots with a control byte equal to h2(hash(key)). The key + // at candidate slot i is compared with key; if key == g.slot(i).key + // we are done and return the slot; if there is an empty slot in the + // group, we stop and return an error; otherwise we continue to the + // next probe index. Tombstones (ctrlDeleted) effectively behave like + // full slots that never match the value we're looking for. + // + // The h2 bits ensure when we compare a key we are likely to have + // actually found the object. That is, the chance is low that keys + // compare false. Thus, when we search for an object, we are unlikely + // to call Equal many times. This likelihood can be analyzed as follows + // (assuming that h2 is a random enough hash function). + // + // Let's assume that there are k "wrong" objects that must be examined + // in a probe sequence. For example, when doing a find on an object + // that is in the table, k is the number of objects between the start + // of the probe sequence and the final found object (not including the + // final found object). The expected number of objects with an h2 match + // is then k/128. Measurements and analysis indicate that even at high + // load factors, k is less than 32, meaning that the number of false + // positive comparisons we must perform is less than 1/8 per find. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotKey, slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return nil, nil, false + } + } +} + +func (t *table) getWithoutKey(typ *abi.SwissMapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return nil, false + } + } +} + +// PutSlot returns a pointer to the element slot where an inserted element +// should be written, and ok if it returned a valid slot. +// +// PutSlot returns ok false if the table was split and the Map needs to find +// the new table. +// +// hash must be the hash of key. +func (t *table) PutSlot(typ *abi.SwissMapType, m *Map, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot we + // find, which we'll use to insert the new entry if necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + t.checkInvariants(typ, m) + return slotElem, true + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + return slotElem, true + } + + t.rehash(typ, m) + return nil, false + } +} + +// uncheckedPutSlot inserts an entry known not to be in the table. +// This is used for grow/split where we are making a new table from +// entries in an existing table. +// +// Decrements growthLeft and increments used. +// +// Requires that the entry does not exist in the table, and that the table has +// room for another element without rehashing. +// +// Requires that there are no deleted entries in the table. +// +// For indirect keys and/or elements, the key and elem pointers can be +// put directly into the map, they do not need to be copied. This +// requires the caller to ensure that the referenced memory never +// changes (by sourcing those pointers from another indirect key/elem +// map). +func (t *table) uncheckedPutSlot(typ *abi.SwissMapType, hash uintptr, key, elem unsafe.Pointer) { + if t.growthLeft == 0 { + panic("invariant failed: growthLeft is unexpectedly 0") + } + + // Given key and its hash hash(key), to insert it, we construct a + // probeSeq, and use it to find the first group with an unoccupied (empty + // or deleted) slot. We place the key/value into the first such slot in + // the group and mark it as full with key's H2. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchEmptyOrDeleted() + if match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + *(*unsafe.Pointer)(slotKey) = key + } else { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + *(*unsafe.Pointer)(slotElem) = elem + } else { + typedmemmove(typ.Elem, slotElem, elem) + } + + t.growthLeft-- + t.used++ + g.ctrls().set(i, ctrl(h2(hash))) + return + } + } +} + +// Delete returns true if it put a tombstone in t. +func (t *table) Delete(typ *abi.SwissMapType, m *Map, hash uintptr, key unsafe.Pointer) bool { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + origSlotKey := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + + if typ.Key.Equal(key, slotKey) { + t.used-- + m.used-- + + if typ.IndirectKey() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(origSlotKey) = nil + } else if typ.Key.Pointers() { + // Only bothing clear the key if there + // are pointers in it. + typedmemclr(typ.Key, slotKey) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(slotElem) = nil + } else { + // Unlike keys, always clear the elem (even if + // it contains no pointers), as compound + // assignment operations depend on cleared + // deleted values. See + // https://go.dev/issue/25936. + typedmemclr(typ.Elem, slotElem) + } + + // Only a full group can appear in the middle + // of a probe sequence (a group with at least + // one empty slot terminates probing). Once a + // group becomes full, it stays full until + // rehashing/resizing. So if the group isn't + // full now, we can simply remove the element. + // Otherwise, we create a tombstone to mark the + // slot as deleted. + var tombstone bool + if g.ctrls().matchEmpty() != 0 { + g.ctrls().set(i, ctrlEmpty) + t.growthLeft++ + } else { + g.ctrls().set(i, ctrlDeleted) + tombstone = true + } + + t.checkInvariants(typ, m) + return tombstone + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return false + } + } +} + +// pruneTombstones goes through the table and tries to remove +// tombstones that are no longer needed. Best effort. +// Note that it only removes tombstones, it does not move elements. +// Moving elements would do a better job but is infeasbile due to +// iterator semantics. +// +// Pruning should only succeed if it can remove O(n) tombstones. +// It would be bad if we did O(n) work to find 1 tombstone to remove. +// Then the next insert would spend another O(n) work to find 1 more +// tombstone to remove, etc. +// +// We really need to remove O(n) tombstones so we can pay for the cost +// of finding them. If we can't, then we need to grow (which is also O(n), +// but guarantees O(n) subsequent inserts can happen in constant time). +func (t *table) pruneTombstones(typ *abi.SwissMapType, m *Map) { + if t.tombstones()*10 < t.capacity { // 10% of capacity + // Not enough tombstones to be worth the effort. + return + } + + // Bit set marking all the groups whose tombstones are needed. + var needed [(maxTableCapacity/abi.MapGroupSlots + 31) / 32]uint32 + + // Trace the probe sequence of every full entry. + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + match := g.ctrls().matchFull() + for match != 0 { + j := match.first() + match = match.removeFirst() + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + if !typ.Key.Equal(key, key) { + // Key not equal to itself. We never have to find these + // keys on lookup (only on iteration), so we can break + // their probe sequences at will. + continue + } + // Walk probe sequence for this key. + // Each tombstone group we need to walk past is marked required. + hash := typ.Hasher(key, m.seed) + for seq := makeProbeSeq(h1(hash), t.groups.lengthMask); ; seq = seq.next() { + if seq.offset == i { + break // reached group of element in probe sequence + } + g := t.groups.group(typ, seq.offset) + m := g.ctrls().matchEmptyOrDeleted() + if m != 0 { // must be deleted, not empty, as we haven't found our key yet + // Mark this group's tombstone as required. + needed[seq.offset/32] |= 1 << (seq.offset % 32) + } + } + } + if g.ctrls().matchEmpty() != 0 { + // Also mark non-tombstone-containing groups, so we don't try + // to remove tombstones from them below. + needed[i/32] |= 1 << (i % 32) + } + } + + // First, see if we can remove enough tombstones to restore capacity. + // This function is O(n), so only remove tombstones if we can remove + // enough of them to justify the O(n) cost. + cnt := 0 + for i := uint64(0); i <= t.groups.lengthMask; i++ { + if needed[i/32]>>(i%32)&1 != 0 { + continue + } + g := t.groups.group(typ, i) + m := g.ctrls().matchEmptyOrDeleted() // must be deleted + cnt += m.count() + } + if cnt*10 < int(t.capacity) { // Can we restore 10% of capacity? + return // don't bother removing tombstones. Caller will grow instead. + } + + // Prune unneeded tombstones. + for i := uint64(0); i <= t.groups.lengthMask; i++ { + if needed[i/32]>>(i%32)&1 != 0 { + continue + } + g := t.groups.group(typ, i) + m := g.ctrls().matchEmptyOrDeleted() // must be deleted + for m != 0 { + k := m.first() + m = m.removeFirst() + g.ctrls().set(k, ctrlEmpty) + t.growthLeft++ + } + // TODO: maybe we could convert all slots at once + // using some bitvector trickery. + } +} + +// tombstones returns the number of deleted (tombstone) entries in the table. A +// tombstone is a slot that has been deleted but is still considered occupied +// so as not to violate the probing invariant. +func (t *table) tombstones() uint16 { + return (t.capacity*maxAvgGroupLoad)/abi.MapGroupSlots - t.used - t.growthLeft +} + +// Clear deletes all entries from the table resulting in an empty table. +func (t *table) Clear(typ *abi.SwissMapType) { + mgl := t.maxGrowthLeft() + if t.used == 0 && t.growthLeft == mgl { // no current entries and no tombstones + return + } + // We only want to do the work of clearing slots + // if they are full. But we also don't want to do too + // much work to figure out whether a slot is full or not, + // especially if clearing a slot is cheap. + // 1) We decide group-by-group instead of slot-by-slot. + // If any slot in a group is full, we zero the whole group. + // 2) If groups are unlikely to be empty, don't bother + // testing for it. + // 3) If groups are 50%/50% likely to be empty, also don't + // bother testing, as it confuses the branch predictor. See #75097. + // 4) But if a group is really large, do the test anyway, as + // clearing is expensive. + fullTest := uint64(t.used)*4 <= t.groups.lengthMask // less than ~0.25 entries per group -> >3/4 empty groups + if typ.SlotSize > 32 { + // For large slots, it is always worth doing the test first. + fullTest = true + } + if fullTest { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + if g.ctrls().anyFull() { + typedmemclr(typ.Group, g.data) + } + g.ctrls().setEmpty() + } + } else { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + typedmemclr(typ.Group, g.data) + g.ctrls().setEmpty() + } + } + t.used = 0 + t.growthLeft = mgl +} + +type Iter struct { + key unsafe.Pointer // Must be in first position. Write nil to indicate iteration end (see cmd/compile/internal/walk/range.go). + elem unsafe.Pointer // Must be in second position (see cmd/compile/internal/walk/range.go). + typ *abi.SwissMapType + m *Map + + // Randomize iteration order by starting iteration at a random slot + // offset. The offset into the directory uses a separate offset, as it + // must adjust when the directory grows. + entryOffset uint64 + dirOffset uint64 + + // Snapshot of Map.clearSeq at iteration initialization time. Used to + // detect clear during iteration. + clearSeq uint64 + + // Value of Map.globalDepth during the last call to Next. Used to + // detect directory grow during iteration. + globalDepth uint8 + + // dirIdx is the current directory index, prior to adjustment by + // dirOffset. + dirIdx int + + // tab is the table at dirIdx during the previous call to Next. + tab *table + + // group is the group at entryIdx during the previous call to Next. + group groupReference + + // entryIdx is the current entry index, prior to adjustment by entryOffset. + // The lower 3 bits of the index are the slot index, and the upper bits + // are the group index. + entryIdx uint64 +} + +// Init initializes Iter for iteration. +func (it *Iter) Init(typ *abi.SwissMapType, m *Map) { + it.typ = typ + + if m == nil || m.used == 0 { + return + } + + dirIdx := 0 + var groupSmall groupReference + if m.dirLen <= 0 { + // Use dirIdx == -1 as sentinel for small maps. + dirIdx = -1 + groupSmall.data = m.dirPtr + } + + it.m = m + it.entryOffset = rand() + it.dirOffset = rand() + it.globalDepth = m.globalDepth + it.dirIdx = dirIdx + it.group = groupSmall + it.clearSeq = m.clearSeq +} + +func (it *Iter) Initialized() bool { + return it.typ != nil +} + +// Map returns the map this iterator is iterating over. +func (it *Iter) Map() *Map { + return it.m +} + +// Key returns a pointer to the current key. nil indicates end of iteration. +// +// Must not be called prior to Next. +func (it *Iter) Key() unsafe.Pointer { + return it.key +} + +// Elem returns a pointer to the current element. nil indicates end of +// iteration. +// +// Must not be called prior to Next. +func (it *Iter) Elem() unsafe.Pointer { + return it.elem +} + +func (it *Iter) nextDirIdx() { + // Skip other entries in the directory that refer to the same + // logical table. There are two cases of this: + // + // Consider this directory: + // + // - 0: *t1 + // - 1: *t1 + // - 2: *t2a + // - 3: *t2b + // + // At some point, the directory grew to accommodate a split of + // t2. t1 did not split, so entries 0 and 1 both point to t1. + // t2 did split, so the two halves were installed in entries 2 + // and 3. + // + // If dirIdx is 0 and it.tab is t1, then we should skip past + // entry 1 to avoid repeating t1. + // + // If dirIdx is 2 and it.tab is t2 (pre-split), then we should + // skip past entry 3 because our pre-split t2 already covers + // all keys from t2a and t2b (except for new insertions, which + // iteration need not return). + // + // We can achieve both of these by using to difference between + // the directory and table depth to compute how many entries + // the table covers. + entries := 1 << (it.m.globalDepth - it.tab.localDepth) + it.dirIdx += entries + it.tab = nil + it.group = groupReference{} + it.entryIdx = 0 +} + +// Return the appropriate key/elem for key at slotIdx index within it.group, if +// any. +func (it *Iter) grownKeyElem(key unsafe.Pointer, slotIdx uintptr) (unsafe.Pointer, unsafe.Pointer, bool) { + newKey, newElem, ok := it.m.getWithKey(it.typ, key) + if !ok { + // Key has likely been deleted, and + // should be skipped. + // + // One exception is keys that don't + // compare equal to themselves (e.g., + // NaN). These keys cannot be looked + // up, so getWithKey will fail even if + // the key exists. + // + // However, we are in luck because such + // keys cannot be updated and they + // cannot be deleted except with clear. + // Thus if no clear has occurred, the + // key/elem must still exist exactly as + // in the old groups, so we can return + // them from there. + // + // TODO(prattmic): Consider checking + // clearSeq early. If a clear occurred, + // Next could always return + // immediately, as iteration doesn't + // need to return anything added after + // clear. + if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { + elem := it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + return key, elem, true + } + + // This entry doesn't exist anymore. + return nil, nil, false + } + + return newKey, newElem, true +} + +// Next proceeds to the next element in iteration, which can be accessed via +// the Key and Elem methods. +// +// The table can be mutated during iteration, though there is no guarantee that +// the mutations will be visible to the iteration. +// +// Init must be called prior to Next. +func (it *Iter) Next() { + if it.m == nil { + // Map was empty at Iter.Init. + it.key = nil + it.elem = nil + return + } + + if it.m.writing != 0 { + fatal("concurrent map iteration and map write") + return + } + + if it.dirIdx < 0 { + // Map was small at Init. + for ; it.entryIdx < abi.MapGroupSlots; it.entryIdx++ { + k := uintptr(it.entryIdx+it.entryOffset) % abi.MapGroupSlots + + if (it.group.ctrls().get(k) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted. + continue + } + + key := it.group.key(it.typ, k) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // As below, if we have grown to a full map since Init, + // we continue to use the old group to decide the keys + // to return, but must look them up again in the new + // tables. + grown := it.m.dirLen > 0 + var elem unsafe.Pointer + if grown { + var ok bool + newKey, newElem, ok := it.m.getWithKey(it.typ, key) + if !ok { + // See comment below. + if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { + elem = it.group.elem(it.typ, k) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } else { + continue + } + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, k) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + it.entryIdx++ + it.key = key + it.elem = elem + return + } + it.key = nil + it.elem = nil + return + } + + if it.globalDepth != it.m.globalDepth { + // Directory has grown since the last call to Next. Adjust our + // directory index. + // + // Consider: + // + // Before: + // - 0: *t1 + // - 1: *t2 <- dirIdx + // + // After: + // - 0: *t1a (split) + // - 1: *t1b (split) + // - 2: *t2 <- dirIdx + // - 3: *t2 + // + // That is, we want to double the current index when the + // directory size doubles (or quadruple when the directory size + // quadruples, etc). + // + // The actual (randomized) dirIdx is computed below as: + // + // dirIdx := (it.dirIdx + it.dirOffset) % it.m.dirLen + // + // Multiplication is associative across modulo operations, + // A * (B % C) = (A * B) % (A * C), + // provided that A is positive. + // + // Thus we can achieve this by adjusting it.dirIdx, + // it.dirOffset, and it.m.dirLen individually. + orders := it.m.globalDepth - it.globalDepth + it.dirIdx <<= orders + it.dirOffset <<= orders + // it.m.dirLen was already adjusted when the directory grew. + + it.globalDepth = it.m.globalDepth + } + + // Continue iteration until we find a full slot. + for ; it.dirIdx < it.m.dirLen; it.nextDirIdx() { + // Resolve the table. + if it.tab == nil { + dirIdx := int((uint64(it.dirIdx) + it.dirOffset) & uint64(it.m.dirLen-1)) + newTab := it.m.directoryAt(uintptr(dirIdx)) + if newTab.index != dirIdx { + // Normally we skip past all duplicates of the + // same entry in the table (see updates to + // it.dirIdx at the end of the loop below), so + // this case wouldn't occur. + // + // But on the very first call, we have a + // completely randomized dirIdx that may refer + // to a middle of a run of tables in the + // directory. Do a one-time adjustment of the + // offset to ensure we start at first index for + // newTable. + diff := dirIdx - newTab.index + it.dirOffset -= uint64(diff) + dirIdx = newTab.index + } + it.tab = newTab + } + + // N.B. Use it.tab, not newTab. It is important to use the old + // table for key selection if the table has grown. See comment + // on grown below. + + entryMask := uint64(it.tab.capacity) - 1 + if it.entryIdx > entryMask { + // Continue to next table. + continue + } + + // Fast path: skip matching and directly check if entryIdx is a + // full slot. + // + // In the slow path below, we perform an 8-slot match check to + // look for full slots within the group. + // + // However, with a max load factor of 7/8, each slot in a + // mostly full map has a high probability of being full. Thus + // it is cheaper to check a single slot than do a full control + // match. + + entryIdx := (it.entryIdx + it.entryOffset) & entryMask + slotIdx := uintptr(entryIdx & (abi.MapGroupSlots - 1)) + if slotIdx == 0 || it.group.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.MapGroupSlotsBits + it.group = it.tab.groups.group(it.typ, groupIdx) + } + + if (it.group.ctrls().get(slotIdx) & ctrlEmpty) == 0 { + // Slot full. + + key := it.group.key(it.typ, slotIdx) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + grown := it.tab.index == -1 + var elem unsafe.Pointer + if grown { + newKey, newElem, ok := it.grownKeyElem(key, slotIdx) + if !ok { + // This entry doesn't exist + // anymore. Continue to the + // next one. + goto next + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + it.entryIdx++ + it.key = key + it.elem = elem + return + } + + next: + it.entryIdx++ + + // Slow path: use a match on the control word to jump ahead to + // the next full slot. + // + // This is highly effective for maps with particularly low load + // (e.g., map allocated with large hint but few insertions). + // + // For maps with medium load (e.g., 3-4 empty slots per group) + // it also tends to work pretty well. Since slots within a + // group are filled in order, then if there have been no + // deletions, a match will allow skipping past all empty slots + // at once. + // + // Note: it is tempting to cache the group match result in the + // iterator to use across Next calls. However because entries + // may be deleted between calls later calls would still need to + // double-check the control value. + + var groupMatch bitset + for it.entryIdx <= entryMask { + entryIdx := (it.entryIdx + it.entryOffset) & entryMask + slotIdx := uintptr(entryIdx & (abi.MapGroupSlots - 1)) + + if slotIdx == 0 || it.group.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.MapGroupSlotsBits + it.group = it.tab.groups.group(it.typ, groupIdx) + } + + if groupMatch == 0 { + groupMatch = it.group.ctrls().matchFull() + + if slotIdx != 0 { + // Starting in the middle of the group. + // Ignore earlier groups. + groupMatch = groupMatch.removeBelow(slotIdx) + } + + // Skip over groups that are composed of only empty or + // deleted slots. + if groupMatch == 0 { + // Jump past remaining slots in this + // group. + it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) + continue + } + + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + if it.entryIdx > entryMask { + // Past the end of this table's iteration. + continue + } + entryIdx += uint64(i - slotIdx) + slotIdx = i + } + + key := it.group.key(it.typ, slotIdx) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // If the table has changed since the last + // call, then it has grown or split. In this + // case, further mutations (changes to + // key->elem or deletions) will not be visible + // in our snapshot table. Instead we must + // consult the new table by doing a full + // lookup. + // + // We still use our old table to decide which + // keys to lookup in order to avoid returning + // the same key twice. + grown := it.tab.index == -1 + var elem unsafe.Pointer + if grown { + newKey, newElem, ok := it.grownKeyElem(key, slotIdx) + if !ok { + // This entry doesn't exist anymore. + // Continue to the next one. + groupMatch = groupMatch.removeFirst() + if groupMatch == 0 { + // No more entries in this + // group. Continue to next + // group. + it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) + continue + } + + // Next full slot. + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + continue + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + // Jump ahead to the next full slot or next group. + groupMatch = groupMatch.removeFirst() + if groupMatch == 0 { + // No more entries in + // this group. Continue + // to next group. + it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) + } else { + // Next full slot. + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + } + + it.key = key + it.elem = elem + return + } + + // Continue to next table. + } + + it.key = nil + it.elem = nil + return +} + +// Replaces the table with one larger table or two split tables to fit more +// entries. Since the table is replaced, t is now stale and should not be +// modified. +func (t *table) rehash(typ *abi.SwissMapType, m *Map) { + // TODO(prattmic): SwissTables typically perform a "rehash in place" + // operation which recovers capacity consumed by tombstones without growing + // the table by reordering slots as necessary to maintain the probe + // invariant while eliminating all tombstones. + // + // However, it is unclear how to make rehash in place work with + // iteration. Since iteration simply walks through all slots in order + // (with random start offset), reordering the slots would break + // iteration. + // + // As an alternative, we could do a "resize" to new groups allocation + // of the same size. This would eliminate the tombstones, but using a + // new allocation, so the existing grow support in iteration would + // continue to work. + + newCapacity := 2 * t.capacity + if newCapacity <= maxTableCapacity { + t.grow(typ, m, newCapacity) + return + } + + t.split(typ, m) +} + +// Bitmask for the last selection bit at this depth. +func localDepthMask(localDepth uint8) uintptr { + if !Use64BitHash { + return uintptr(1) << (32 - localDepth) + } + return uintptr(1) << (64 - localDepth) +} + +// split the table into two, installing the new tables in the map directory. +func (t *table) split(typ *abi.SwissMapType, m *Map) { + localDepth := t.localDepth + localDepth++ + + // TODO: is this the best capacity? + left := newTable(typ, maxTableCapacity, -1, localDepth) + right := newTable(typ, maxTableCapacity, -1, localDepth) + + // Split in half at the localDepth bit from the top. + mask := localDepthMask(localDepth) + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted + continue + } + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, j) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + var newTable *table + if hash&mask == 0 { + newTable = left + } else { + newTable = right + } + newTable.uncheckedPutSlot(typ, hash, key, elem) + } + } + + m.installTableSplit(t, left, right) + t.index = -1 +} + +// grow the capacity of the table by allocating a new table with a bigger array +// and uncheckedPutting each element of the table into the new table (we know +// that no insertion here will Put an already-present value), and discard the +// old table. +func (t *table) grow(typ *abi.SwissMapType, m *Map, newCapacity uint16) { + newTable := newTable(typ, uint64(newCapacity), t.index, t.localDepth) + + if t.capacity > 0 { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted + continue + } + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, j) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + + newTable.uncheckedPutSlot(typ, hash, key, elem) + } + } + } + + newTable.checkInvariants(typ, m) + m.replaceTable(newTable) + t.index = -1 +} + +// probeSeq maintains the state for a probe sequence that iterates through the +// groups in a table. The sequence is a triangular progression of the form +// hash, hash + 1, hash + 1 + 2, hash + 1 + 2 + 3, ..., modulo mask + 1. +// The i-th term of the sequence is +// +// p(i) := hash + (i^2 + i)/2 (mod mask+1) +// +// The sequence effectively outputs the indexes of *groups*. The group +// machinery allows us to check an entire group with minimal branching. +// +// It turns out that this probe sequence visits every group exactly once if +// the number of groups is a power of two, since (i^2+i)/2 is a bijection in +// Z/(2^m). See https://en.wikipedia.org/wiki/Quadratic_probing +type probeSeq struct { + mask uint64 + offset uint64 + index uint64 +} + +func makeProbeSeq(hash uintptr, mask uint64) probeSeq { + return probeSeq{ + mask: mask, + offset: uint64(hash) & mask, + index: 0, + } +} + +func (s probeSeq) next() probeSeq { + s.index++ + s.offset = (s.offset + s.index) & s.mask + return s +} + +func (t *table) clone(typ *abi.SwissMapType) *table { + // Shallow copy the table structure. + t2 := new(table) + *t2 = *t + t = t2 + + // We need to just deep copy the groups.data field. + oldGroups := t.groups + newGroups := newGroups(typ, oldGroups.lengthMask+1) + for i := uint64(0); i <= oldGroups.lengthMask; i++ { + oldGroup := oldGroups.group(typ, i) + newGroup := newGroups.group(typ, i) + cloneGroup(typ, newGroup, oldGroup) + } + t.groups = newGroups + + return t +} diff --git a/runtime/internal/runtime/maps/table_debug.go b/runtime/internal/runtime/maps/table_debug.go new file mode 100644 index 0000000000..5c52bd11ef --- /dev/null +++ b/runtime/internal/runtime/maps/table_debug.go @@ -0,0 +1,131 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/abi" +) + +const debugLog = false + +func (t *table) checkInvariants(typ *abi.SwissMapType, m *Map) { + if !debugLog { + return + } + + // For every non-empty slot, verify we can retrieve the key using Get. + // Count the number of used and deleted slots. + var used uint16 + var deleted uint16 + var empty uint16 + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + c := g.ctrls().get(j) + switch { + case c == ctrlDeleted: + deleted++ + case c == ctrlEmpty: + empty++ + default: + used++ + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // Can't lookup keys that don't compare equal + // to themselves (e.g., NaN). + if !typ.Key.Equal(key, key) { + continue + } + + if _, ok := t.Get(typ, m, key); !ok { + hash := typ.Hasher(key, m.seed) + print("invariant failed: slot(", i, "/", j, "): key ") + dump(key, typ.Key.Size_) + print(" not found [hash=", hash, ", h2=", h2(hash), " h1=", h1(hash), "]\n") + t.Print(typ, m) + panic("invariant failed: slot: key not found") + } + } + } + } + + if used != t.used { + print("invariant failed: found ", used, " used slots, but used count is ", t.used, "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched used slot count") + } + + growthLeft := (t.capacity*maxAvgGroupLoad)/abi.MapGroupSlots - t.used - deleted + if growthLeft != t.growthLeft { + print("invariant failed: found ", t.growthLeft, " growthLeft, but expected ", growthLeft, "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched growthLeft") + } + if deleted != t.tombstones() { + print("invariant failed: found ", deleted, " tombstones, but expected ", t.tombstones(), "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched tombstones") + } + + if empty == 0 { + print("invariant failed: found no empty slots (violates probe invariant)\n") + t.Print(typ, m) + panic("invariant failed: found no empty slots (violates probe invariant)") + } +} +func (t *table) Print(typ *abi.SwissMapType, m *Map) { + print(`table{ + index: `, t.index, ` + localDepth: `, t.localDepth, ` + capacity: `, t.capacity, ` + used: `, t.used, ` + growthLeft: `, t.growthLeft, ` + groups: +`) + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + print("\t\tgroup ", i, "\n") + + g := t.groups.group(typ, i) + ctrls := g.ctrls() + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + print("\t\t\tslot ", j, "\n") + + c := ctrls.get(j) + print("\t\t\t\tctrl ", c) + switch c { + case ctrlEmpty: + print(" (empty)\n") + case ctrlDeleted: + print(" (deleted)\n") + default: + print("\n") + } + + print("\t\t\t\tkey ") + dump(g.key(typ, j), typ.Key.Size_) + println("") + print("\t\t\t\telem ") + dump(g.elem(typ, j), typ.Elem.Size_) + println("") + } + } +} + +// TODO(prattmic): not in hex because print doesn't have a way to print in hex +// outside the runtime. +func dump(ptr unsafe.Pointer, size uintptr) { + for size > 0 { + print(*(*byte)(ptr), " ") + ptr = unsafe.Pointer(uintptr(ptr) + 1) + size-- + } +} diff --git a/runtime/internal/runtime/math/math.go b/runtime/internal/runtime/math/math.go index 0b0782b1d0..7fa9d78bcf 100644 --- a/runtime/internal/runtime/math/math.go +++ b/runtime/internal/runtime/math/math.go @@ -2,7 +2,14 @@ package math import "github.com/goplus/llgo/runtime/internal/runtime/goarch" -const MaxUintptr = ^uintptr(0) +const ( + MaxUint16 = ^uint16(0) + MaxUint32 = ^uint32(0) + MaxUint64 = ^uint64(0) + MaxUintptr = ^uintptr(0) + + MaxInt64 = int64(MaxUint64 >> 1) +) // MulUintptr returns a * b and whether the multiplication overflowed. // On supported platforms this is an intrinsic lowered by the compiler. diff --git a/runtime/internal/runtime/sys/intrinsics.go b/runtime/internal/runtime/sys/intrinsics.go new file mode 100644 index 0000000000..9785cd1184 --- /dev/null +++ b/runtime/internal/runtime/sys/intrinsics.go @@ -0,0 +1,256 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys + +// Copied from math/bits to avoid dependence. + +var deBruijn32tab = [32]byte{ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, +} + +const deBruijn32 = 0x077CB531 + +var deBruijn64tab = [64]byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +const deBruijn64 = 0x03f79d71b4ca8b09 + +const ntz8tab = "" + + "\x08\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x06\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x07\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x06\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + +// TrailingZeros32 returns the number of trailing zero bits in x; the result is 32 for x == 0. +func TrailingZeros32(x uint32) int { + if x == 0 { + return 32 + } + // see comment in TrailingZeros64 + return int(deBruijn32tab[(x&-x)*deBruijn32>>(32-5)]) +} + +// TrailingZeros64 returns the number of trailing zero bits in x; the result is 64 for x == 0. +func TrailingZeros64(x uint64) int { + if x == 0 { + return 64 + } + // If popcount is fast, replace code below with return popcount(^x & (x - 1)). + // + // x & -x leaves only the right-most bit set in the word. Let k be the + // index of that bit. Since only a single bit is set, the value is two + // to the power of k. Multiplying by a power of two is equivalent to + // left shifting, in this case by k bits. The de Bruijn (64 bit) constant + // is such that all six bit, consecutive substrings are distinct. + // Therefore, if we have a left shifted version of this constant we can + // find by how many bits it was shifted by looking at which six bit + // substring ended up at the top of the word. + // (Knuth, volume 4, section 7.3.1) + return int(deBruijn64tab[(x&-x)*deBruijn64>>(64-6)]) +} + +// TrailingZeros8 returns the number of trailing zero bits in x; the result is 8 for x == 0. +func TrailingZeros8(x uint8) int { + return int(ntz8tab[x]) +} + +const len8tab = "" + + "\x00\x01\x02\x02\x03\x03\x03\x03\x04\x04\x04\x04\x04\x04\x04\x04" + + "\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + + "\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06" + + "\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + +// Len64 returns the minimum number of bits required to represent x; the result is 0 for x == 0. +// +// nosplit because this is used in src/runtime/histogram.go, which make run in sensitive contexts. +// +//go:nosplit +func Len64(x uint64) (n int) { + if x >= 1<<32 { + x >>= 32 + n = 32 + } + if x >= 1<<16 { + x >>= 16 + n += 16 + } + if x >= 1<<8 { + x >>= 8 + n += 8 + } + return n + int(len8tab[x]) +} + +// --- OnesCount --- + +const m0 = 0x5555555555555555 // 01010101 ... +const m1 = 0x3333333333333333 // 00110011 ... +const m2 = 0x0f0f0f0f0f0f0f0f // 00001111 ... + +// OnesCount64 returns the number of one bits ("population count") in x. +func OnesCount64(x uint64) int { + // Implementation: Parallel summing of adjacent bits. + // See "Hacker's Delight", Chap. 5: Counting Bits. + // The following pattern shows the general approach: + // + // x = x>>1&(m0&m) + x&(m0&m) + // x = x>>2&(m1&m) + x&(m1&m) + // x = x>>4&(m2&m) + x&(m2&m) + // x = x>>8&(m3&m) + x&(m3&m) + // x = x>>16&(m4&m) + x&(m4&m) + // x = x>>32&(m5&m) + x&(m5&m) + // return int(x) + // + // Masking (& operations) can be left away when there's no + // danger that a field's sum will carry over into the next + // field: Since the result cannot be > 64, 8 bits is enough + // and we can ignore the masks for the shifts by 8 and up. + // Per "Hacker's Delight", the first line can be simplified + // more, but it saves at best one instruction, so we leave + // it alone for clarity. + const m = 1<<64 - 1 + x = x>>1&(m0&m) + x&(m0&m) + x = x>>2&(m1&m) + x&(m1&m) + x = (x>>4 + x) & (m2 & m) + x += x >> 8 + x += x >> 16 + x += x >> 32 + return int(x) & (1<<7 - 1) +} + +// LeadingZeros64 returns the number of leading zero bits in x; the result is 64 for x == 0. +func LeadingZeros64(x uint64) int { return 64 - Len64(x) } + +// LeadingZeros8 returns the number of leading zero bits in x; the result is 8 for x == 0. +func LeadingZeros8(x uint8) int { return 8 - Len8(x) } + +// Len8 returns the minimum number of bits required to represent x; the result is 0 for x == 0. +func Len8(x uint8) int { + return int(len8tab[x]) +} + +// Bswap64 returns its input with byte order reversed +// 0x0102030405060708 -> 0x0807060504030201 +func Bswap64(x uint64) uint64 { + c8 := uint64(0x00ff00ff00ff00ff) + a := x >> 8 & c8 + b := (x & c8) << 8 + x = a | b + c16 := uint64(0x0000ffff0000ffff) + a = x >> 16 & c16 + b = (x & c16) << 16 + x = a | b + c32 := uint64(0x00000000ffffffff) + a = x >> 32 & c32 + b = (x & c32) << 32 + x = a | b + return x +} + +// Bswap32 returns its input with byte order reversed +// 0x01020304 -> 0x04030201 +func Bswap32(x uint32) uint32 { + c8 := uint32(0x00ff00ff) + a := x >> 8 & c8 + b := (x & c8) << 8 + x = a | b + c16 := uint32(0x0000ffff) + a = x >> 16 & c16 + b = (x & c16) << 16 + x = a | b + return x +} + +// Prefetch prefetches data from memory addr to cache +// +// AMD64: Produce PREFETCHT0 instruction +// +// ARM64: Produce PRFM instruction with PLDL1KEEP option +func Prefetch(addr uintptr) {} + +// PrefetchStreamed prefetches data from memory addr, with a hint that this data is being streamed. +// That is, it is likely to be accessed very soon, but only once. If possible, this will avoid polluting the cache. +// +// AMD64: Produce PREFETCHNTA instruction +// +// ARM64: Produce PRFM instruction with PLDL1STRM option +func PrefetchStreamed(addr uintptr) {} + +// GetCallerPC returns the program counter (PC) of its caller's caller. +// GetCallerSP returns the stack pointer (SP) of its caller's caller. +// Both are implemented as intrinsics on every platform. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := GetCallerPC() +// sp := GetCallerSP() +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to GetCallerPC and GetCallerSP must be done in the +// frame being asked about. +// +// The result of GetCallerSP is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of GetCallerSP should be used +// immediately and can only be passed to nosplit functions. + +//func GetCallerPC() uintptr + +//func GetCallerSP() uintptr + +// GetClosurePtr returns the pointer to the current closure. +// GetClosurePtr can only be used in an assignment statement +// at the entry of a function. Moreover, go:nosplit directive +// must be specified at the declaration of caller function, +// so that the function prolog does not clobber the closure register. +// for example: +// +// //go:nosplit +// func f(arg1, arg2, arg3 int) { +// dx := GetClosurePtr() +// } +// +// The compiler rewrites calls to this function into instructions that fetch the +// pointer from a well-known register (DX on x86 architecture, etc.) directly. +// +// WARNING: PGO-based devirtualization cannot detect that caller of +// GetClosurePtr requires closure context, and thus must maintain a list of +// these functions, which is in +// cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. +//func GetClosurePtr() uintptr