From 664d8cfb91d5f9ff6d4b52c884a292262e60c208 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 10 Aug 2026 06:05:08 -0700 Subject: [PATCH 1/4] Heap: freeze the current heap as an immortal startup snapshot, and keep snapshot cells' pages clean Heap::freezeCurrentHeapAsImmortalStartupSnapshot() runs after a synchronous full collection and marks every MarkedBlock and precise allocation immortal: the mark bits it has at that point become its permanent liveness, and the collector never marks into, sweeps, allocates from or ages such a block again (BlockDirectory keeps them out of the empty/canAllocate/unswept sets; areMarksStale() answers "not stale" for them, so the stock mark fast paths see live snapshot cells as already marked and never write). The write barrier records snapshot cells that get written in a side set instead of touching their cell state; those cells, plus the precise allocations that existed at the freeze, are the full-collection roots for everything reachable from the snapshot, so a restored process's collections stay proportional to what it allocated itself. Things that would otherwise write into snapshot cells during ordinary execution move out of them: snapshot Structures are flagged with a spare bitfield bit and lock through a striped side lock; snapshot UnlinkedCodeBlocks are not aged; watchpoint sets that live in the snapshot chain their new watchpoints through a side table; every string in the snapshot is made static, and static strings no longer write their refcount; a few identity writes (setIndexInSpace, resetAge, JITStubRoutine flags) are skipped when the value is unchanged. Unlinked code blocks decoded for snapshot executables after the freeze are kept in a heap-side table. Costs on paths every process runs: one byte test in areMarksStale() per mark and the isStatic() test in StringImpl::ref()/deref(); everything else is behind hasImmortalBlocks() or a per-object bit. Options: verboseStartupSnapshotFreeze, unlinkedCodeBlockJettisonAge and useUnlinkedCodeBlockJettisoningForBytecodeCache (defaults preserve current behaviour). --- CLAUDE.md | 4 + .../bytecode/UnlinkedCodeBlock.cpp | 34 ++- .../bytecode/UnlinkedCodeBlock.h | 6 +- Source/JavaScriptCore/bytecode/Watchpoint.cpp | 62 ++++- Source/JavaScriptCore/bytecode/Watchpoint.h | 10 +- Source/JavaScriptCore/heap/BlockDirectory.cpp | 45 ++- Source/JavaScriptCore/heap/BlockDirectory.h | 3 + .../JavaScriptCore/heap/BlockDirectoryBits.h | 1 + Source/JavaScriptCore/heap/Heap.cpp | 259 +++++++++++++++++- Source/JavaScriptCore/heap/Heap.h | 50 ++++ Source/JavaScriptCore/heap/HeapInlines.h | 11 + Source/JavaScriptCore/heap/IsoSubspace.h | 8 + .../JavaScriptCore/heap/JITStubRoutineSet.cpp | 10 +- Source/JavaScriptCore/heap/LocalAllocator.h | 3 + Source/JavaScriptCore/heap/MarkedBlock.cpp | 28 +- Source/JavaScriptCore/heap/MarkedBlock.h | 17 +- Source/JavaScriptCore/heap/MarkedBlockSet.h | 3 + Source/JavaScriptCore/heap/MarkedSpace.cpp | 25 +- Source/JavaScriptCore/heap/MarkedSpace.h | 10 + .../JavaScriptCore/heap/PreciseAllocation.cpp | 9 + .../JavaScriptCore/heap/PreciseAllocation.h | 11 +- Source/JavaScriptCore/heap/SlotVisitor.cpp | 17 +- Source/JavaScriptCore/heap/SlotVisitor.h | 3 + Source/JavaScriptCore/heap/WeakSet.h | 8 +- .../JavaScriptCore/runtime/ButterflyInlines.h | 4 +- Source/JavaScriptCore/runtime/OptionsList.h | 7 + Source/JavaScriptCore/runtime/Structure.cpp | 94 ++++--- Source/JavaScriptCore/runtime/Structure.h | 15 +- .../JavaScriptCore/runtime/StructureInlines.h | 20 +- Source/WTF/wtf/text/StringImpl.h | 8 +- 30 files changed, 702 insertions(+), 83 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d074a3c4ca4e..c92f9ed9d416 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,6 +149,10 @@ High-performance memory allocator with: - Stack trace improvements - Better error reporting for development +5. **Startup Snapshots** + - `heap/Heap.cpp` (`freezeCurrentHeapAsImmortalStartupSnapshot`, `resetPacingAfterSnapshotRestore`), `runtime/VM.cpp` (`didRestoreFromStartupSnapshot`) + - Freezes the live heap as immortal so an embedder can write the process's memory out and resume later launches from it + ### USE_BUN_EVENT_LOOP Custom event loop implementation for Bun's runtime requirements diff --git a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp index eb9089fee154..fd4cda0904ca 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp +++ b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp @@ -26,6 +26,7 @@ #include "config.h" #include "UnlinkedCodeBlock.h" +#include "HeapInlines.h" #include "BaselineJITCode.h" #include "BytecodeLivenessAnalysis.h" @@ -99,9 +100,14 @@ void UnlinkedCodeBlock::visitChildrenImpl(JSCell* cell, Visitor& visitor) UnlinkedCodeBlock* thisObject = uncheckedDowncast(cell); ASSERT_GC_OBJECT_INHERITS(thisObject, info()); Base::visitChildren(thisObject, visitor); - Locker locker { thisObject->cellLock() }; - if (visitor.isFirstVisit()) - thisObject->m_age = std::min(static_cast(thisObject->m_age) + 1, maxAge); + // Snapshot (immortal) code blocks are immutable and never jettisoned: no lock, no aging, so their pages stay clean. + bool isSnapshot = Heap::isStartupSnapshotCell(thisObject); + std::optional> locker; + if (!isSnapshot) { + locker.emplace(thisObject->cellLock()); + if (visitor.isFirstVisit()) + thisObject->m_age = std::min(static_cast(thisObject->m_age) + 1, maxAge); + } for (auto& barrier : thisObject->m_functionDecls) visitor.append(barrier); for (auto& barrier : thisObject->m_functionExprs) @@ -109,9 +115,9 @@ void UnlinkedCodeBlock::visitChildrenImpl(JSCell* cell, Visitor& visitor) visitor.appendValues(thisObject->m_constantRegisters.span()); size_t extraMemory = thisObject->metadataSizeInBytes(); if (thisObject->m_instructions) - extraMemory += thisObject->m_instructions->sizeInBytes(); + extraMemory += thisObject->m_instructions->ownedSizeInBytes(); if (thisObject->hasRareData()) - extraMemory += thisObject->m_rareData->sizeInBytes(locker); + extraMemory += locker ? thisObject->m_rareData->sizeInBytes(*locker) : thisObject->m_rareData->sizeInBytes(NoLockingNecessary); if (thisObject->m_expressionInfo) extraMemory += thisObject->m_expressionInfo->byteSize(); extraMemory += thisObject->m_jumpTargets.byteSize(); @@ -348,4 +354,22 @@ void UnlinkedCodeBlock::allocateSharedProfiles(unsigned numBinaryArithProfiles, m_unaryArithProfiles = FixedVector(numUnaryArithProfiles); } +#if USE(BUN_JSC_ADDITIONS) +UnlinkedCodeBlock::ComponentSizes UnlinkedCodeBlock::componentSizesForCensus() +{ + ComponentSizes r { }; + if (m_instructions) + r.instructions = m_instructions->sizeInBytes(); + if (m_expressionInfo) + r.expressionInfo = m_expressionInfo->byteSize(); + r.metadata = m_metadata->sizeInBytesForGC(); + r.identifiers = m_identifiers.size() * sizeof(Identifier); + r.constants = m_constantRegisters.size() * sizeof(WriteBarrier) + m_constantsSourceCodeRepresentation.size() * sizeof(SourceCodeRepresentation); + r.jumpTargets = m_jumpTargets.size() * sizeof(JSInstructionStream::Offset); + r.profiles = m_valueProfiles.size() * sizeof(UnlinkedValueProfile) + m_arrayProfiles.size() * sizeof(UnlinkedArrayProfile) + m_binaryArithProfiles.size() * sizeof(BinaryArithProfile) + m_unaryArithProfiles.size() * sizeof(UnaryArithProfile); + if (m_rareData) + r.rareData = m_rareData->sizeInBytes(NoLockingNecessary); + return r; +} +#endif } // namespace JSC diff --git a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h index d557c2125294..e57c5413e9c9 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h +++ b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h @@ -172,6 +172,10 @@ class UnlinkedCodeBlock : public JSCell { bool isBuiltinDefaultClassConstructor() const { return m_isBuiltinDefaultClassConstructor; } bool hasExpressionInfo() { return !m_expressionInfo->isEmpty(); } +#if USE(BUN_JSC_ADDITIONS) + struct ComponentSizes { size_t instructions { 0 }, expressionInfo { 0 }, metadata { 0 }, identifiers { 0 }, constants { 0 }, jumpTargets { 0 }, profiles { 0 }, rareData { 0 }; }; + JS_EXPORT_PRIVATE ComponentSizes componentSizesForCensus(); // memory attribution tooling +#endif bool hasCheckpoints() const { return m_hasCheckpoints; } void setHasCheckpoints() { m_hasCheckpoints = true; } @@ -324,7 +328,7 @@ class UnlinkedCodeBlock : public JSCell { static constexpr unsigned maxAge = 7; unsigned age() const { return m_age; } - void resetAge() { m_age = 0; } + void resetAge() { if (m_age) m_age = 0; } // conditional so a snapshot cell that is already young is not written NeedsClassFieldInitializer needsClassFieldInitializer() const { diff --git a/Source/JavaScriptCore/bytecode/Watchpoint.cpp b/Source/JavaScriptCore/bytecode/Watchpoint.cpp index 66f84efb1fe7..57dee4c807a5 100644 --- a/Source/JavaScriptCore/bytecode/Watchpoint.cpp +++ b/Source/JavaScriptCore/bytecode/Watchpoint.cpp @@ -25,6 +25,10 @@ #include "config.h" #include "Watchpoint.h" +#if USE(BUN_JSC_ADDITIONS) +#include +#include +#endif #include "AdaptiveInferredPropertyValueWatchpointBase.h" #include "CachedSpecialPropertyAdaptiveStructureWatchpoint.h" @@ -101,6 +105,39 @@ WatchpointSet::WatchpointSet(WatchpointState state) { } + +using WatchpointList = SentinelLinkedList>; +#if USE(BUN_JSC_ADDITIONS) +// Snapshot: watchpoints added to a WatchpointSet that lives inside the snapshot go on a per-set side chain in fresh memory, +// so the push doesn't splice into (and dirty) the snapshotd neighbours/sentinel. State bytes still change in the set itself +// (JIT'd code reads them inline); the list is only ever walked here. +static Lock s_snapshotSideChainsLock; +static UncheckedKeyHashMap>& snapshotSideChains() WTF_REQUIRES_LOCK(s_snapshotSideChainsLock) +{ + static NeverDestroyed>> map; + return map; +} +static ALWAYS_INLINE bool isSnapshotWatchpointSet(WatchpointSet* set) +{ + return Heap::isInSnapshotImmortalRange(set); +} +WatchpointList* WatchpointSet::snapshotSideChain(bool createIfMissing) +{ + if (!isSnapshotWatchpointSet(this)) + return nullptr; + Locker locker { s_snapshotSideChainsLock }; + auto& map = snapshotSideChains(); + if (createIfMissing) { + auto& slot = map.add(this, nullptr).iterator->value; + if (!slot) + slot = makeUniqueWithoutFastMallocCheck(); + return slot.get(); + } + auto it = map.find(this); + return it == map.end() ? nullptr : it->value.get(); +} +#endif + WatchpointSet::~WatchpointSet() { // FIXME(rdar://165379969): This is here to silence a RefcountDebugger ASSERT. But the @@ -113,6 +150,12 @@ WatchpointSet::~WatchpointSet() // either keeping the watchpoint set's owner alive, or does some weak reference thing. while (!m_set.isEmpty()) m_set.begin()->remove(); +#if USE(BUN_JSC_ADDITIONS) + if (auto* side = snapshotSideChain(false)) { + while (!side->isEmpty()) + side->begin()->remove(); + } +#endif } void WatchpointSet::add(Watchpoint* watchpoint) @@ -121,9 +164,14 @@ void WatchpointSet::add(Watchpoint* watchpoint) ASSERT(state() != IsInvalidated); if (!watchpoint) return; - m_set.push(watchpoint); - m_setIsNotEmpty = true; - m_state = IsWatched; + if (auto* side = snapshotSideChain(true)) + side->push(watchpoint); + else + m_set.push(watchpoint); + if (!m_setIsNotEmpty) + m_setIsNotEmpty = true; + if (m_state != IsWatched) + m_state = IsWatched; } void WatchpointSet::fireAllSlow(VM& vm, const FireDetail& detail) @@ -164,8 +212,9 @@ void WatchpointSet::fireAllWatchpoints(VM& vm, const FireDetail& detail) // The safest thing to do is to DeferGCForAWhile to prevent this GC from happening. DeferGCForAWhile deferGC(vm); - while (!m_set.isEmpty()) { - Watchpoint& watchpoint = *m_set.begin(); + WatchpointList* side = snapshotSideChain(false); + while (!m_set.isEmpty() || (side && !side->isEmpty())) { + Watchpoint& watchpoint = !m_set.isEmpty() ? *m_set.begin() : *side->begin(); ASSERT(watchpoint.isOnList()); // Removing the Watchpoint before firing it makes it possible to implement watchpoints @@ -178,7 +227,6 @@ void WatchpointSet::fireAllWatchpoints(VM& vm, const FireDetail& detail) // possible to add itself to the transition watchpoint set of the singleton object's new // Structure. watchpoint.remove(); - ASSERT(&*m_set.begin() != &watchpoint); ASSERT(!watchpoint.isOnList()); watchpoint.fire(vm, detail); @@ -191,6 +239,8 @@ void WatchpointSet::take(WatchpointSet* other) { ASSERT(state() == ClearWatchpoint); m_set.takeFrom(other->m_set); + if (auto* otherSide = other->snapshotSideChain(false)) + m_set.takeFrom(*otherSide); m_setIsNotEmpty = other->m_setIsNotEmpty; m_state = other->m_state; other->m_setIsNotEmpty = false; diff --git a/Source/JavaScriptCore/bytecode/Watchpoint.h b/Source/JavaScriptCore/bytecode/Watchpoint.h index 27eaafa0dc5f..ec44c7520239 100644 --- a/Source/JavaScriptCore/bytecode/Watchpoint.h +++ b/Source/JavaScriptCore/bytecode/Watchpoint.h @@ -279,6 +279,11 @@ class WatchpointSet : public ThreadSafeRefCounted { int8_t m_setIsNotEmpty; SentinelLinkedList> m_set; +#if USE(BUN_JSC_ADDITIONS) + SentinelLinkedList>* snapshotSideChain(bool createIfMissing); // snapshot: post-restore additions to an snapshotd set +#else + static constexpr SentinelLinkedList>* snapshotSideChain(bool) { return nullptr; } +#endif }; // InlineWatchpointSet is a low-overhead, non-copyable watchpoint set in which @@ -357,7 +362,8 @@ class InlineWatchpointSet { protect(fat())->fireAll(vm, fireDetails); return; } - if (decodeState(m_data) == ClearWatchpoint) + // Already-invalidated thin sets are terminal: skip the identity store (keeps clean/shared pages clean). + if (decodeState(m_data) != IsWatched) return; m_data = encodeState(IsInvalidated); WTF::storeStoreFence(); @@ -367,7 +373,7 @@ class InlineWatchpointSet { { if (isFat()) protect(fat())->invalidate(vm, detail); - else + else if (decodeState(m_data) != IsInvalidated) m_data = encodeState(IsInvalidated); } diff --git a/Source/JavaScriptCore/heap/BlockDirectory.cpp b/Source/JavaScriptCore/heap/BlockDirectory.cpp index 93ac2023997e..3c2584268185 100644 --- a/Source/JavaScriptCore/heap/BlockDirectory.cpp +++ b/Source/JavaScriptCore/heap/BlockDirectory.cpp @@ -264,6 +264,8 @@ void BlockDirectory::lastChanceToFinalize() { forEachBlock( [&] (MarkedBlock::Handle* block) { + if (block->block().isImmortal()) [[unlikely]] + return; // snapshot memory is abandoned, never finalized: the cells belong to every launch, not to this VM block->lastChanceToFinalize(); }); } @@ -286,6 +288,11 @@ void BlockDirectory::beginMarkingForFullCollection() // as the next one is eden. markingNotEmptyBits().clearAll(); markingRetiredBits().clearAll(); +#if USE(BUN_JSC_ADDITIONS) + // Immortal blocks are never marked into, but must still be iterated by forEachMarkedCell (unconditional finalizers etc.). + markingNotEmptyBits() |= immortalBits(); + markingRetiredBits() |= immortalBits(); +#endif } void BlockDirectory::endMarking() @@ -308,8 +315,8 @@ void BlockDirectory::endMarking() // vectors. // Sweeper is suspended so we don't need the lock here. - emptyBits() = liveBits() & ~markingNotEmptyBits(); - canAllocateBits() = liveBits() & ~markingRetiredBits(); + emptyBits() = liveBits() & ~markingNotEmptyBits() & ~immortalBits(); + canAllocateBits() = liveBits() & ~markingRetiredBits() & ~immortalBits(); switch (m_attributes.destruction) { case NeedsDestruction: { @@ -342,15 +349,45 @@ void BlockDirectory::endMarking() void BlockDirectory::snapshotUnsweptForEdenCollection() { assertSweeperIsSuspended(); - unsweptBits() |= edenBits(); + unsweptBits() |= edenBits() & ~immortalBits(); } void BlockDirectory::snapshotUnsweptForFullCollection() { assertSweeperIsSuspended(); - unsweptBits() = liveBits(); + unsweptBits() = liveBits() & ~immortalBits(); } +#if USE(BUN_JSC_ADDITIONS) +void BlockDirectory::makeAllBlocksImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion) +{ + // Return any block held by an allocator — stopAllocating() records exactly the handed-out cells as newlyAllocated + // (the free-list remainder stays dead) — then make allocators forget it. + { + Locker allocatorsLocker { m_localAllocatorsLock }; + m_localAllocators.forEach([&](LocalAllocator* allocator) { + allocator->stopAllocating(); + allocator->forgetCurrentBlock(); + }); + } + Locker locker(bitvectorLock()); + assertSweeperIsSuspended(); + for (size_t index = 0; index < m_blocks.size(); ++index) { + if (!isLive(index) || !m_blocks[index]) + continue; + m_blocks[index]->block().makeImmortal(markingVersion, newlyAllocatedVersion); + setIsImmortal(index, true); + setIsMarkingNotEmpty(index, true); + setIsMarkingRetired(index, true); + setIsEmpty(index, false); + setIsCanAllocate(index, false); + setIsUnswept(index, false); + setIsEden(index, false); + setIsAllocated(index, false); + } +} +#endif + MarkedBlock::Handle* BlockDirectory::findBlockToSweep(unsigned& unsweptCursor) { Locker locker(bitvectorLock()); diff --git a/Source/JavaScriptCore/heap/BlockDirectory.h b/Source/JavaScriptCore/heap/BlockDirectory.h index a26bde2325b2..f4ed6310608e 100644 --- a/Source/JavaScriptCore/heap/BlockDirectory.h +++ b/Source/JavaScriptCore/heap/BlockDirectory.h @@ -70,6 +70,9 @@ class BlockDirectory { void endMarking(); void NODELETE snapshotUnsweptForEdenCollection(); void NODELETE snapshotUnsweptForFullCollection(); +#if USE(BUN_JSC_ADDITIONS) + void makeAllBlocksImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion); +#endif void sweep(); void shrink(); void assertNoUnswept(); diff --git a/Source/JavaScriptCore/heap/BlockDirectoryBits.h b/Source/JavaScriptCore/heap/BlockDirectoryBits.h index ab7753e77fdc..c2398ec5c5f8 100644 --- a/Source/JavaScriptCore/heap/BlockDirectoryBits.h +++ b/Source/JavaScriptCore/heap/BlockDirectoryBits.h @@ -43,6 +43,7 @@ namespace JSC { macro(eden, Eden) /* The set of all blocks that have new objects since the last GC. */\ macro(unswept, Unswept) /* The set of all blocks that could be swept by the incremental sweeper. */\ macro(inUse, InUse) /* This tells us if a block is currently being allocated from or swept. This acts like a lock bit. */\ + macro(immortal, Immortal) /* startup-snapshot blocks; only ever set under USE(BUN_JSC_ADDITIONS), so the masks below are no-ops otherwise */\ \ /* These are computed during marking. */\ macro(markingNotEmpty, MarkingNotEmpty) /* The set of all blocks that are not empty. */ \ diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index 0a4e2038f374..14991bb17f07 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -20,6 +20,9 @@ #include "config.h" #include "Heap.h" +#if USE(BUN_JSC_ADDITIONS) +#include "VerifierSlotVisitor.h" +#endif #include "JSCJSValueInlines.h" @@ -122,6 +125,9 @@ #include "JSFFICallback.h" #include "JSFFIFunction.h" #include "JSString.h" +#if USE(BUN_JSC_ADDITIONS) +#include "StructureInlines.h" +#endif #include #endif @@ -1033,6 +1039,12 @@ void Heap::endMarking() m_objectSpace.endMarking(); setMutatorShouldBeFenced(Options::forceFencedBarrier()); +#if USE(BUN_JSC_ADDITIONS) + if (m_objectSpace.hasImmortalBlocks()) { + Locker locker { m_snapshotRememberedLock }; + m_snapshotRememberedThisCycle.clear(); + } +#endif } size_t Heap::objectCount() @@ -1183,6 +1195,13 @@ void Heap::deleteAllUnlinkedCodeBlocks(DeleteAllCodeEffort effort) if (m_collectionScope && effort == DeleteAllCodeIfNotCollecting) return; +#if USE(BUN_JSC_ADDITIONS) + { + Locker locker { m_snapshotRememberedLock }; + m_snapshotUnlinkedCodeBlocks.clear(); + } +#endif + VM& vm = this->vm(); PreventCollectionScope preventCollectionScope(*this); @@ -1192,6 +1211,10 @@ void Heap::deleteAllUnlinkedCodeBlocks(DeleteAllCodeEffort effort) unlinkedFunctionExecutableSpaceAndSet.set.forEachLiveCell( [&] (HeapCell* cell, HeapCell::Kind) { UnlinkedFunctionExecutable* executable = static_cast(cell); +#if USE(BUN_JSC_ADDITIONS) + if (isStartupSnapshotCell(executable)) [[unlikely]] + return; // never written; whatever it gained after the restore was in the side table cleared above +#endif executable->clearCode(vm); }); @@ -1220,12 +1243,84 @@ void Heap::deleteUnmarkedCompiledCode() m_jitStubRoutines->deleteUnmarkedJettisonedStubRoutines(vm()); } +#if USE(BUN_JSC_ADDITIONS) +UnlinkedFunctionCodeBlock* Heap::snapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable* executable, CodeSpecializationKind kind) +{ + Locker locker { m_snapshotRememberedLock }; + auto it = m_snapshotUnlinkedCodeBlocks.find({ executable, static_cast(kind) }); + return it == m_snapshotUnlinkedCodeBlocks.end() ? nullptr : it->value; +} + +void Heap::setSnapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable* executable, CodeSpecializationKind kind, UnlinkedFunctionCodeBlock* codeBlock) +{ + Locker locker { m_snapshotRememberedLock }; + if (codeBlock) + m_snapshotUnlinkedCodeBlocks.set({ executable, static_cast(kind) }, codeBlock); + else + m_snapshotUnlinkedCodeBlocks.remove({ executable, static_cast(kind) }); +} + +void Heap::resetPacingAfterSnapshotRestore() +{ + // The limits in the snapshot were computed when the snapshot cells were ordinary live cells; here they are immortal and + // never counted, so those limits would let everything allocated after the restore accumulate in fresh blocks before + // the first collection. Start over as an empty heap does. + m_sizeAfterLastCollect = 0; + m_sizeAfterLastFullCollect = 0; + m_sizeAfterLastEdenCollect = 0; + m_extraMemorySize = 0; // reported by cells that are now immortal; a full collection would drop it too + m_deprecatedExtraMemorySize = 0; + // First cycle triggers on the mortal live size alone (immortal cells are not counted), never later than this heap type's own floor. + m_maxHeapSize = std::min(minHeapSize(m_heapType, m_ramSize), Options::mediumHeapSize()); + m_maxEdenSize = m_maxHeapSize; +} + + +void Heap::evacuateTablesForStartupSnapshot() +{ + m_objectSpace.blocks().evacuateStorage(); + rehomeOutOfSnapshotPages(m_weakGCHashTables); + { + Locker locker { m_snapshotRememberedLock }; + rehomeOutOfSnapshotPages(m_snapshotWrittenEver); + decltype(m_snapshotUnlinkedCodeBlocks) fresh; + m_snapshotUnlinkedCodeBlocks.swap(fresh); // empty at snapshot time anyway + } + if (auto* cache = vm().megamorphicCache()) + cache->age(CollectionScope::Full); // bumps the epoch (entries become misses) without rewriting the snapshotd entry arrays +} + + + +uintptr_t Heap::s_snapshotImmortalRangeLo = 0; +uintptr_t Heap::s_snapshotImmortalRangeSpan = 0; + +void Heap::rememberSnapshotCell(JSCell* cell) +{ + // Snapshot cells stay PossiblyBlack forever (their header is never written); dedupe per cycle via the side set. + Locker locker { m_snapshotRememberedLock }; + m_snapshotWrittenEver.add(cell); + if (m_snapshotRememberedThisCycle.add(cell).isNewEntry) + m_mutatorMarkStack->append(cell); +} + +void Heap::willVisitSnapshotCell(JSCell* cell) +{ + Locker locker { m_snapshotRememberedLock }; + m_snapshotRememberedThisCycle.remove(cell); +} +#endif + void Heap::addToRememberedSet(const JSCell* constCell) { JSCell* cell = const_cast(constCell); ASSERT(cell); ASSERT(!Options::useConcurrentJIT() || !isCompilationThread()); m_barriersExecuted++; + if (m_objectSpace.hasImmortalBlocks() && isStartupSnapshotCell(cell)) [[unlikely]] { + rememberSnapshotCell(cell); + return; + } if (m_mutatorShouldBeFenced) { WTF::loadLoadFence(); if (!isMarked(cell)) { @@ -3064,8 +3159,167 @@ constexpr bool samplingProfilerSupported = false; static UNUSED_FUNCTION void visitSamplingProfiler(VM&, AbstractSlotVisitor&) { }; #endif +#if USE(BUN_JSC_ADDITIONS) +void Heap::freezeCurrentHeapAsImmortalStartupSnapshot() +{ + RELEASE_ASSERT(vm().currentThreadIsHoldingAPILock()); + vm().completeAllJITPlansBeforeStartupSnapshot(); // no compiler thread may be reading structures while prepareForStartupSnapshot() switches their lock + { + // Settle lazily-materialized state that would otherwise be written into snapshot cells after freeze: + // every Structure gets its property table now (and the freeze GC below keeps them). + Vector structures; + { + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (isJSCellKind(kind) && static_cast(heapCell)->type() == StructureType) + structures.append(static_cast(static_cast(heapCell))); + return IterationStatus::Continue; + }); + } // materializing tables allocates, which iteration forbids + DeferGC deferGC(vm()); + size_t withTable = 0; + for (Structure* structure : structures) { + structure->prepareForStartupSnapshot(vm()); + if (structure->hasPropertyTableForSnapshot()) + withTable++; + } + dataLogLnIf(Options::verboseStartupSnapshotFreeze(), "[imm] pre-materialized property tables: ", withTable, " of ", structures.size(), " structures have a table"); + } + collectNow(Sync, CollectionScope::Full); + if (Options::verboseStartupSnapshotFreeze()) [[unlikely]] { + size_t withTable = 0, total = 0; + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (isJSCellKind(kind) && static_cast(heapCell)->type() == StructureType) { + total++; + if (static_cast(static_cast(heapCell))->hasPropertyTableForSnapshot()) + withTable++; + } + return IterationStatus::Continue; + }); + dataLogLn("[imm] after freeze GC: ", withTable, " of ", total, " structures have a table"); + } + PreventCollectionScope preventCollectionScope(*this); + sweeper().stopSweeping(); // nothing sweeps immortal blocks; a pending incremental sweep would rewrite their pages + m_objectSpace.freezeAllBlocksAsImmortal(); + m_objectSpace.forEachSubspace([](Subspace& subspace) { + if (subspace.isIsoSubspace()) + static_cast(subspace).abandonLowerTierPreciseFreeListForSnapshot(); + return IterationStatus::Continue; + }); + for (PreciseAllocation* allocation : m_objectSpace.preciseAllocations()) { + if (!allocation->isLive()) + continue; + allocation->makeImmortal(); + if (isJSCellKind(allocation->attributes().cellKind)) { + JSCell* cell = static_cast(allocation->cell()); + m_snapshotPreciseRoots.append(cell); + if (cell->cellState() != CellState::PossiblyBlack) + cell->setCellState(CellState::PossiblyBlack); // so stores into it take the barrier slow path (-> side remembered set) like every other snapshot cell + } + } + { + // Cells still PossiblyGrey (remembered around the freeze GC) would get blackened on their next visit; settle them now + // (they stay on the mutator mark stack, so nothing is lost) so snapshot headers are never written afterwards. + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (isJSCellKind(kind) && !heapCell->isPreciseAllocation() && heapCell->markedBlock().isImmortal()) { + JSCell* cell = static_cast(heapCell); + if (cell->cellState() != CellState::PossiblyBlack) { + rememberSnapshotCell(cell); + cell->setCellState(CellState::PossiblyBlack); + } + } + return IterationStatus::Continue; + }); + } + if (Options::verboseStartupSnapshotFreeze()) [[unlikely]] { + size_t states[4] = { 0, 0, 0, 0 }; + HashMap whiteByClass; + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (!isJSCellKind(kind) || heapCell->isPreciseAllocation() || !heapCell->markedBlock().isImmortal()) + return IterationStatus::Continue; + auto st = static_cast(static_cast(heapCell)->cellState()); + states[std::min(st, 3u)]++; + if (st == static_cast(CellState::DefinitelyWhite)) + whiteByClass.add(static_cast(heapCell)->className(), 0).iterator->value++; + return IterationStatus::Continue; + }); + dataLogLn("[imm] frozen cellState histogram: PossiblyBlack=", states[0], " DefinitelyWhite=", states[1], " PossiblyGrey=", states[2], " other=", states[3]); + for (auto& e : whiteByClass) + dataLogLn(" white: ", e.key, " x", e.value); + } + { + size_t count = 0, atoms = 0; + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (!isJSCellKind(kind)) + return IterationStatus::Continue; + JSCell* cell = static_cast(heapCell); + if (cell->isString()) { + if (auto* impl = static_cast(cell)->tryGetValueImpl()) { + impl->settleLazyHeaderWritesForStartupSnapshot(); + impl->makeStaticForSnapshot(); + count++; + } + } + return IterationStatus::Continue; + }); + for (auto& entry : vm().atomStringTable()->table()) { + if (StringImpl* impl = entry.get()) { + impl->settleLazyHeaderWritesForStartupSnapshot(); + impl->makeStaticForSnapshot(); + atoms++; + } + } + dataLogLnIf(Options::verboseStartupSnapshotFreeze(), "[imm] made ", count, " string cells' StringImpls static, and ", atoms, " atom-table entries (overlapping sets)"); + } +} +#endif + void Heap::addCoreConstraints() { +#if USE(BUN_JSC_ADDITIONS) + m_constraintSet->add( + "Img", "Immortal snapshot roots", + MAKE_MARKING_CONSTRAINT_EXECUTOR_PAIR(([this] (auto& visitor) { + if (!m_objectSpace.hasImmortalBlocks()) + return; + Vector redecoded; + { + Locker locker { m_snapshotRememberedLock }; + redecoded = copyToVector(m_snapshotUnlinkedCodeBlocks.values()); + } + for (auto* codeBlock : redecoded) + visitor.appendUnbarriered(JSValue(static_cast(codeBlock))); + // Unwritten snapshot cells can only reference snapshot cells or the precise allocations that existed at freeze; + // so the Full-GC roots are: every snapshot cell written since freeze (side card set) + those precise allocations. + if (m_collectionScope != CollectionScope::Full) + return; + SetRootMarkReasonScope rootScope(visitor, RootMarkReason::StrongReferences); + Vector written; + { + Locker locker { m_snapshotRememberedLock }; + written = copyToVector(m_snapshotWrittenEver); + } + for (JSCell* cell : written) { + if constexpr (std::is_same_v, SlotVisitor>) + visitor.visitImmortalCellAsRoot(cell); + else + visitor.appendUnbarriered(JSValue(cell)); + } + for (JSCell* cell : m_snapshotPreciseRoots) { // immortal too: appendUnbarriered would see "already marked" and never look inside + if constexpr (std::is_same_v, SlotVisitor>) + visitor.visitImmortalCellAsRoot(cell); + else + visitor.appendUnbarriered(JSValue(cell)); + } + })), + ConstraintVolatility::GreyedByExecution); +#endif + + m_constraintSet->add( "Cs", "Conservative Scan", MAKE_MARKING_CONSTRAINT_EXECUTOR_PAIR(([this, lastVersion = static_cast(0)] (auto& visitor) mutable { @@ -3505,7 +3759,10 @@ void Heap::verifyGC() if (Heap::isMarked(cell)) return; - dataLogLn("\n" "GC Verifier: ERROR cell ", RawPointer(cell), " was not marked"); + dataLogLn("\n" "GC Verifier: ERROR cell ", RawPointer(cell), " was not marked", + " type=", isJSCellKind(cell->cellKind()) ? static_cast(cell)->className() : "aux", + " immortalBlock=", (!cell->isPreciseAllocation() && cell->markedBlock().isImmortal()), + " cellState=", isJSCellKind(cell->cellKind()) ? static_cast(static_cast(cell)->cellState()) : -1); if (Options::verboseVerifyGC()) [[unlikely]] visitor.dumpMarkerData(cell); RELEASE_ASSERT(this->isMarked(cell)); diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h index bcf1df069af1..8ac47c0c7405 100644 --- a/Source/JavaScriptCore/heap/Heap.h +++ b/Source/JavaScriptCore/heap/Heap.h @@ -23,6 +23,7 @@ #include "ArrayBuffer.h" #include "CellState.h" +#include "CodeSpecializationKind.h" #include "CollectionScope.h" #include "CollectorPhase.h" #include "CompleteSubspace.h" @@ -104,6 +105,11 @@ class SpaceTimeMutatorScheduler; class StopIfNecessaryTimer; class SweepingScope; class VM; +class StructureTransitionTable; +class StructureChain; +class Structure; +class UnlinkedFunctionExecutable; +class UnlinkedFunctionCodeBlock; class VerifierSlotVisitor; class WeakGCHashTable; struct CurrentThreadState; @@ -478,6 +484,42 @@ class Heap { JS_EXPORT_PRIVATE bool unprotect(JSValue); // True when the protect count drops to 0. JS_EXPORT_PRIVATE size_t extraMemorySize(); // Non-GC memory referenced by GC objects. +#if USE(BUN_JSC_ADDITIONS) + // Snapshot: after a synchronous full collection, freeze every MarkedBlock as an immortal snapshot block. + JS_EXPORT_PRIVATE void freezeCurrentHeapAsImmortalStartupSnapshot(); + static inline bool isStartupSnapshotCell(const JSCell*); + // Rule 2 (link state out of snapshot cells): decoded UnlinkedFunctionCodeBlocks for snapshot UnlinkedFunctionExecutables live here, not in the (never written) cell. + UnlinkedFunctionCodeBlock* snapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind); + void setSnapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind, UnlinkedFunctionCodeBlock*); + void clearSnapshotUnlinkedCodeBlocks() + { + Locker locker { m_snapshotRememberedLock }; + m_snapshotUnlinkedCodeBlocks.clear(); + } + // A table that lives on in the restored process gets fresh backing storage: growing it in place would dirty snapshot pages. + template static void rehomeOutOfSnapshotPages(Table& table) + { + Table copy = table; + table.swap(copy); + } + void evacuateTablesForStartupSnapshot(); // after restore: re-home per-heap tables that take inserts every run + JS_EXPORT_PRIVATE void resetPacingAfterSnapshotRestore(); + // Barrier slow path for snapshot owners: side remembered set instead of cellState writes. + void rememberSnapshotCell(JSCell*); + // [lo, lo+span) is the mapped snapshot's allocator range (span 0: none); the embedder sets it at restore. Slow paths only. + JS_EXPORT_PRIVATE static uintptr_t s_snapshotImmortalRangeLo; + JS_EXPORT_PRIVATE static uintptr_t s_snapshotImmortalRangeSpan; + static bool isInSnapshotImmortalRange(const void* p) { return reinterpret_cast(p) - s_snapshotImmortalRangeLo < s_snapshotImmortalRangeSpan; } + // Called by the visitor right before it reads a snapshot cell's fields, so a racing store re-remembers it. + void willVisitSnapshotCell(JSCell*); +#else + static bool isStartupSnapshotCell(const JSCell*) { return false; } + static bool isInSnapshotImmortalRange(const void*) { return false; } + UnlinkedFunctionCodeBlock* snapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind) { return nullptr; } + void setSnapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind, UnlinkedFunctionCodeBlock*) { } + void rememberSnapshotCell(JSCell*) { } + void willVisitSnapshotCell(JSCell*) { } +#endif JS_EXPORT_PRIVATE size_t size(); JS_EXPORT_PRIVATE size_t capacity(); JS_EXPORT_PRIVATE size_t objectCount(); @@ -902,6 +944,14 @@ class Heap { std::unique_ptr m_collectorSlotVisitor; std::unique_ptr m_mutatorSlotVisitor; std::unique_ptr m_mutatorMarkStack; +#if USE(BUN_JSC_ADDITIONS) + // Prototype card table: snapshot cells written since freeze (sticky, Full-GC roots) and this cycle (barrier dedupe). + Lock m_snapshotRememberedLock; + UncheckedKeyHashSet m_snapshotWrittenEver WTF_GUARDED_BY_LOCK(m_snapshotRememberedLock); + UncheckedKeyHashMap, UnlinkedFunctionCodeBlock*> m_snapshotUnlinkedCodeBlocks WTF_GUARDED_BY_LOCK(m_snapshotRememberedLock); // visited as roots by the Img constraint + UncheckedKeyHashSet m_snapshotRememberedThisCycle WTF_GUARDED_BY_LOCK(m_snapshotRememberedLock); + Vector m_snapshotPreciseRoots; // filled once at freeze with the world stopped, read-only afterwards: no lock +#endif std::unique_ptr m_raceMarkStack; std::unique_ptr m_constraintSet; std::unique_ptr m_verifierSlotVisitor; diff --git a/Source/JavaScriptCore/heap/HeapInlines.h b/Source/JavaScriptCore/heap/HeapInlines.h index 9c6647adfee2..cace8a5b8709 100644 --- a/Source/JavaScriptCore/heap/HeapInlines.h +++ b/Source/JavaScriptCore/heap/HeapInlines.h @@ -56,6 +56,17 @@ inline JSC::Heap* Heap::heap(const JSValue v) return heap(v.asCell()); } +#if USE(BUN_JSC_ADDITIONS) +ALWAYS_INLINE bool Heap::isStartupSnapshotCell(const JSCell* cell) +{ + if (!cell) + return false; + if (cell->isPreciseAllocation()) + return cell->preciseAllocation().isImmortal(); + return cell->markedBlock().isImmortal(); +} +#endif + // Defined in Heap.cpp with NEVER_INLINE to prevent LTO from breaking compiler barriers // ALWAYS_INLINE bool Heap::isMarked(const void* rawCell) // { diff --git a/Source/JavaScriptCore/heap/IsoSubspace.h b/Source/JavaScriptCore/heap/IsoSubspace.h index b5e9bd58ee49..b144bc3f66ec 100644 --- a/Source/JavaScriptCore/heap/IsoSubspace.h +++ b/Source/JavaScriptCore/heap/IsoSubspace.h @@ -54,6 +54,14 @@ class IsoSubspace final : public Subspace { void* tryAllocateLowerTierPrecise(size_t cellSize); void destroyLowerTierPreciseFreeList(); +#if USE(BUN_JSC_ADDITIONS) + // The free cells live in snapshot pages: never hand them out again. + void abandonLowerTierPreciseFreeListForSnapshot() + { + while (!m_lowerTierPreciseFreeList.isEmpty()) + m_lowerTierPreciseFreeList.remove(&*m_lowerTierPreciseFreeList.begin()); + } +#endif void sweep(); diff --git a/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp b/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp index 98ce80022f30..35e6274daa68 100644 --- a/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp +++ b/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp @@ -95,8 +95,10 @@ void JITStubRoutineSet::prepareForConservativeScan() void JITStubRoutineSet::clearMarks() { // Immutable code routines do not matter. - for (auto& entry : m_routines) - entry.routine->m_mayBeExecuting = false; + for (auto& entry : m_routines) { + if (entry.routine->m_mayBeExecuting) + entry.routine->m_mayBeExecuting = false; + } } void JITStubRoutineSet::markSlow(uintptr_t address) @@ -135,8 +137,8 @@ void JITStubRoutineSet::deleteUnmarkedJettisonedStubRoutines(VM& vm) ASSERT(vm.heap.isInPhase(CollectorPhase::End)); auto shouldRemove = [&](GCAwareJITStubRoutine* stub) { - if (!stub->m_ownerIsDead) - stub->m_ownerIsDead = stub->removeDeadOwners(vm); + if (!stub->m_ownerIsDead && stub->removeDeadOwners(vm)) + stub->m_ownerIsDead = true; // If the stub is running right now, we should keep it alive regardless of whether owner CodeBlock gets dead. // It is OK since we already marked all the related cells. diff --git a/Source/JavaScriptCore/heap/LocalAllocator.h b/Source/JavaScriptCore/heap/LocalAllocator.h index 37727a06d827..054ab0581262 100644 --- a/Source/JavaScriptCore/heap/LocalAllocator.h +++ b/Source/JavaScriptCore/heap/LocalAllocator.h @@ -51,6 +51,9 @@ class LocalAllocator : public BasicRawSentinelNode { void prepareForAllocation(); void resumeAllocating(); void stopAllocatingForGood(); +#if USE(BUN_JSC_ADDITIONS) + void forgetCurrentBlock() { m_freeList.clear(); m_currentBlock = nullptr; m_lastActiveBlock = nullptr; } +#endif static constexpr ptrdiff_t offsetOfFreeList(); static constexpr ptrdiff_t offsetOfCellSize(); diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp index 4430cbd053e2..94d7094d9aec 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.cpp +++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp @@ -47,6 +47,8 @@ namespace JSC { // compiler barriers (loadLoadFence/compilerFence) on x86_64. NEVER_INLINE bool MarkedBlock::isMarked(HeapVersion markingVersion, const void* p) { + if (isImmortal()) [[unlikely]] + return header().m_marks.get(atomNumber(p)); HeapVersion version; Dependency dependency = Dependency::loadAndFence(&header().m_markingVersion, version); if (version != markingVersion) [[unlikely]] @@ -54,10 +56,28 @@ NEVER_INLINE bool MarkedBlock::isMarked(HeapVersion markingVersion, const void* return header().m_marks.concurrentGet(atomNumber(p), dependency); } +#if USE(BUN_JSC_ADDITIONS) +void MarkedBlock::makeImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion) +{ + // Runs right after a synchronous full GC + sweep: m_marks (if current) ∪ newlyAllocated (if current) is exactly the + // live set; every other slot is a zapped free cell that nothing references. That bitmap is frozen here and never + // written again, so the stock mark fast paths are safe on snapshot blocks: a live snapshot cell reads "already marked" + // (write-free), and a dead slot is unreachable by precise marking (conservative roots filter through isLive()). + if (header().m_markingVersion != markingVersion) + header().m_marks.clearAll(); + if (header().m_newlyAllocatedVersion == newlyAllocatedVersion) + header().m_marks.merge(header().m_newlyAllocated); + header().m_markingVersion = markingVersion; + header().m_isImmortal = true; +} +#endif + // NEVER_INLINE to prevent LTO from inlining this function, which can break // compiler barriers (Dependency::fence/loadLoadFence/compilerFence) on x86_64. NEVER_INLINE bool MarkedBlock::Handle::isLive(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion, bool isMarking, const HeapCell* cell) { + if (block().isImmortal()) [[unlikely]] + return block().isMarkedRaw(cell); m_directory->assertIsMutatorOrMutatorIsStopped(); if (m_directory->isAllocated(this)) return true; @@ -333,6 +353,8 @@ inline void MarkedBlock::setupTestForDumpInfoAndCrash() { } void MarkedBlock::aboutToMarkSlow(HeapVersion markingVersion, HeapCell* cell) { + if (isImmortal()) [[unlikely]] + return; ASSERT(vm().heap.objectSpace().isMarking()); setupTestForDumpInfoAndCrash(); @@ -420,7 +442,7 @@ void MarkedBlock::resetMarks() #if ASSERT_ENABLED void MarkedBlock::assertMarksNotStale() { - ASSERT(header().m_markingVersion == vm().heap.objectSpace().markingVersion()); + ASSERT(isImmortal() || header().m_markingVersion == vm().heap.objectSpace().markingVersion()); } #endif // ASSERT_ENABLED @@ -548,6 +570,10 @@ Subspace* MarkedBlock::Handle::subspace() const void MarkedBlock::Handle::sweep(FreeList* freeList) { + if (block().isImmortal()) [[unlikely]] { + dataLogLn("[imm] BUG: sweeping immortal block ", RawPointer(&block()), " toFreeList=", !!freeList, " dir empty=", m_directory->isEmpty(this), " canAlloc=", m_directory->isCanAllocate(this), " unswept=", m_directory->isUnswept(this), " immortalBit=", m_directory->isImmortal(this)); + CRASH(); + } SweepingScope sweepingScope(*heap()); m_directory->assertIsMutatorOrMutatorIsStopped(); ASSERT(m_directory->isInUse(this)); diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h index 1bf8d98a2bae..b108b660690d 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.h +++ b/Source/JavaScriptCore/heap/MarkedBlock.h @@ -313,6 +313,10 @@ class MarkedBlock { WTF::BitSet m_marks; WTF::BitSet m_newlyAllocated; void* m_verifierMemo { nullptr }; +#if USE(BUN_JSC_ADDITIONS) + // Immortal (snapshot) block: liveness is frozen in m_marks, the collector never writes here, never sweeps or allocates. + bool m_isImmortal { false }; +#endif }; private: @@ -403,6 +407,12 @@ class MarkedBlock { bool isMarkedRaw(const void* p); HeapVersion markingVersion() const { return header().m_markingVersion; } +#if USE(BUN_JSC_ADDITIONS) + bool isImmortal() const { return header().m_isImmortal; } + void makeImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion); +#else + constexpr bool isImmortal() const { return false; } +#endif const WTF::BitSet& marks() const; @@ -584,6 +594,8 @@ inline unsigned MarkedBlock::atomNumber(const void* p) inline bool MarkedBlock::areMarksStale(HeapVersion markingVersion) { + if (isImmortal()) [[unlikely]] + return false; // a snapshot block keeps the version it was frozen with; its marks are its liveness return markingVersion != header().m_markingVersion; } @@ -592,7 +604,7 @@ inline Dependency MarkedBlock::aboutToMark(HeapVersion markingVersion, HeapCell* HeapVersion version; Dependency dependency = Dependency::loadAndFence(&header().m_markingVersion, version); if (version != markingVersion) [[unlikely]] - aboutToMarkSlow(markingVersion, cell); + aboutToMarkSlow(markingVersion, cell); // snapshot blocks keep a stale version forever; the slow path returns without writing return dependency; } @@ -624,6 +636,9 @@ inline bool MarkedBlock::isMarked(const void* p, Dependency dependency) inline bool MarkedBlock::testAndSetMarked(const void* p, Dependency dependency) { + // Snapshot blocks need no case here: every snapshot cell a precise pointer can reach is live, so its frozen bit is set and + // concurrentTestAndSet returns without writing; dead snapshot cells are only ever proposed by conservative scanning, + // which consults isMarked (areMarksStale) first. assertMarksNotStale(); return header().m_marks.concurrentTestAndSet(atomNumber(p), dependency); } diff --git a/Source/JavaScriptCore/heap/MarkedBlockSet.h b/Source/JavaScriptCore/heap/MarkedBlockSet.h index a75958e41066..943fccf37068 100644 --- a/Source/JavaScriptCore/heap/MarkedBlockSet.h +++ b/Source/JavaScriptCore/heap/MarkedBlockSet.h @@ -40,6 +40,9 @@ class MarkedBlockSet { TinyBloomFilter filter() const; const UncheckedKeyHashSet& set() const; +#if USE(BUN_JSC_ADDITIONS) + void evacuateStorage() { auto copy = m_set; m_set.swap(copy); } // snapshot: move the table to fresh pages so inserts don't dirty snapshotd ones +#endif private: void recomputeFilter(); diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp index 369ca6660700..9ebab02dd637 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.cpp +++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp @@ -265,7 +265,8 @@ void MarkedSpace::sweepPreciseAllocations() } continue; } - allocation->setIndexInSpace(dstIndex); + if (allocation->indexInSpace() != dstIndex) // snapshot allocations form a stable prefix: no store to their headers + allocation->setIndexInSpace(dstIndex); m_preciseAllocations[dstIndex++] = allocation; } m_preciseAllocations.shrinkCapacity(dstIndex); @@ -344,7 +345,8 @@ void MarkedSpace::prepareForConservativeScan() }); unsigned index = m_preciseAllocationsOffsetForThisCollection; for (auto* start = m_preciseAllocationsForThisCollectionBegin; start != m_preciseAllocationsForThisCollectionEnd; ++start, ++index) { - (*start)->setIndexInSpace(index); + if ((*start)->indexInSpace() != index) + (*start)->setIndexInSpace(index); ASSERT(m_preciseAllocations[index] == *start); ASSERT(m_preciseAllocations[index]->indexInSpace() == index); } @@ -399,6 +401,19 @@ MarkedBlock::Handle* MarkedSpace::findMarkedBlockHandleDebug(MarkedBlock* block) return result; } +#if USE(BUN_JSC_ADDITIONS) +void MarkedSpace::freezeAllBlocksAsImmortal() +{ + HeapVersion newlyAllocatedVersion = this->newlyAllocatedVersion(); + HeapVersion markingVersion = this->markingVersion(); + forEachDirectory([&](BlockDirectory& directory) -> IterationStatus { + directory.makeAllBlocksImmortal(markingVersion, newlyAllocatedVersion); + return IterationStatus::Continue; + }); + m_hasImmortalBlocks = true; +} +#endif + void MarkedSpace::freeBlock(MarkedBlock::Handle* block) { m_capacity -= MarkedBlock::blockSize; @@ -468,8 +483,10 @@ void MarkedSpace::endMarking() m_newlyAllocatedVersion = nextVersion(m_newlyAllocatedVersion); - for (unsigned i = m_preciseAllocationsOffsetForThisCollection; i < m_preciseAllocations.size(); ++i) - m_preciseAllocations[i]->clearNewlyAllocated(); + for (unsigned i = m_preciseAllocationsOffsetForThisCollection; i < m_preciseAllocations.size(); ++i) { + if (m_preciseAllocations[i]->isNewlyAllocated()) // same: an unconditional store would dirty every snapshot allocation's header + m_preciseAllocations[i]->clearNewlyAllocated(); + } if (ASSERT_ENABLED) { for (PreciseAllocation* allocation : m_preciseAllocations) diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h index 1c22af51cddc..4277aa5161e7 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.h +++ b/Source/JavaScriptCore/heap/MarkedSpace.h @@ -127,6 +127,13 @@ class MarkedSpace { template void forEachLiveCell(HeapIterationScope&, const Functor&); template void forEachDeadCell(HeapIterationScope&, const Functor&); template void forEachBlock(const Functor&); +#if USE(BUN_JSC_ADDITIONS) + // Snapshot: freeze every current block as an immortal snapshot block (call with the world stopped, right after a Full GC). + void freezeAllBlocksAsImmortal(); + bool hasImmortalBlocks() const { return m_hasImmortalBlocks; } +#else + constexpr bool hasImmortalBlocks() const { return false; } +#endif template void forEachSubspace(const Functor&); void shrink(); @@ -218,6 +225,9 @@ class MarkedSpace { HeapVersion m_newlyAllocatedVersion { initialVersion }; HeapVersion m_edenVersion { initialVersion }; bool m_isIterating { false }; +#if USE(BUN_JSC_ADDITIONS) + bool m_hasImmortalBlocks { false }; +#endif bool m_isMarking { false }; bool m_conservativeScanIsPrepared { false }; Lock m_directoryLock; diff --git a/Source/JavaScriptCore/heap/PreciseAllocation.cpp b/Source/JavaScriptCore/heap/PreciseAllocation.cpp index f439bd312ddb..12347188976a 100644 --- a/Source/JavaScriptCore/heap/PreciseAllocation.cpp +++ b/Source/JavaScriptCore/heap/PreciseAllocation.cpp @@ -122,6 +122,9 @@ PreciseAllocation* PreciseAllocation::tryReallocate(size_t size, Subspace* subsp unsigned oldAdjustment = m_adjustment; void* oldBasePointer = basePointer(); +#if USE(BUN_JSC_ADDITIONS) + ASSERT(!isImmortal()); // ButterflyInlines never reallocates a snapshot allocation +#endif void* newSpace = subspace->alignedMemoryAllocator()->tryReallocateMemory(oldBasePointer, adjustedAlignmentAllocationSize); if (!newSpace) return nullptr; @@ -232,6 +235,8 @@ PreciseAllocation::~PreciseAllocation() void PreciseAllocation::lastChanceToFinalize() { + if (isImmortal()) [[unlikely]] + return; // as for immortal blocks: abandoned, not finalized m_weakSet.lastChanceToFinalize(); clearMarked(); clearNewlyAllocated(); @@ -259,6 +264,8 @@ void PreciseAllocation::flip() // The dead object newly created before this 1 0 => 1 0 => 1 0 => 0 0 => dead // ^ // This is ensured since this function is used only for full GC. + if (isImmortal()) + return; m_isNewlyAllocated |= isMarked(); m_isMarked.store(false, std::memory_order_relaxed); } @@ -271,6 +278,8 @@ bool PreciseAllocation::isEmpty() void PreciseAllocation::sweep() { m_weakSet.sweep(); + if (isImmortal()) + return; if (m_hasValidCell && !isLive()) { if (m_attributes.destruction != DoesNotNeedDestruction) diff --git a/Source/JavaScriptCore/heap/PreciseAllocation.h b/Source/JavaScriptCore/heap/PreciseAllocation.h index a6eee51bada3..3f7003304771 100644 --- a/Source/JavaScriptCore/heap/PreciseAllocation.h +++ b/Source/JavaScriptCore/heap/PreciseAllocation.h @@ -87,7 +87,13 @@ class PreciseAllocation : public BasicRawSentinelNode { void NODELETE flip(); bool isNewlyAllocated() const { return m_isNewlyAllocated; } - ALWAYS_INLINE bool isMarked() { return m_isMarked.load(std::memory_order_relaxed); } + ALWAYS_INLINE bool isMarked() { return isImmortal() || m_isMarked.load(std::memory_order_relaxed); } +#if USE(BUN_JSC_ADDITIONS) + void makeImmortal() { m_isImmortal = true; } + bool isImmortal() const { return m_isImmortal; } +#else + constexpr bool isImmortal() const { return false; } +#endif ALWAYS_INLINE bool isMarked(HeapCell*) { return isMarked(); } ALWAYS_INLINE bool isMarked(HeapCell*, Dependency) { return isMarked(); } ALWAYS_INLINE bool isMarked(HeapVersion, HeapCell*) { return isMarked(); } @@ -173,6 +179,9 @@ class PreciseAllocation : public BasicRawSentinelNode { size_t m_cellSize; bool m_isNewlyAllocated : 1; bool m_hasValidCell : 1; +#if USE(BUN_JSC_ADDITIONS) + bool m_isImmortal : 1 { false }; +#endif // Worst case adjustment needed would be halfAlignment + portionOfObjectThatMustFitInCacheLine // which is 8 + 16 -> 24 bytes i.e. will fit in 5 bits. If we need more bits in the future, we // can also encode this number of uintptr_t words to save 3 bits. diff --git a/Source/JavaScriptCore/heap/SlotVisitor.cpp b/Source/JavaScriptCore/heap/SlotVisitor.cpp index 75082f85b03a..fa4162b71ed3 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.cpp +++ b/Source/JavaScriptCore/heap/SlotVisitor.cpp @@ -344,6 +344,18 @@ class SetCurrentCellScope { SlotVisitor& m_visitor; }; +#if USE(BUN_JSC_ADDITIONS) +void SlotVisitor::visitImmortalCellAsRoot(const JSCell* cell) +{ + // Like visitChildren, but never writes the (frozen, already-black) header, so clean snapshot pages stay clean. + SetCurrentCellScope currentCellScope(*this, cell); + m_isFirstVisit = false; + m_heap.willVisitSnapshotCell(const_cast(cell)); + WTF::storeLoadFence(); + cell->methodTable()->visitChildren(const_cast(cell), *this); +} +#endif + ALWAYS_INLINE void SlotVisitor::visitChildren(const JSCell* cell) { ASSERT(m_heap.isMarked(cell)); @@ -362,7 +374,10 @@ ALWAYS_INLINE void SlotVisitor::visitChildren(const JSCell* cell) // not clear to me that it would be correct or profitable to bail here if the object is already // black. - cell->setCellState(CellState::PossiblyBlack); + if (cell->cellState() != CellState::PossiblyBlack) + cell->setCellState(CellState::PossiblyBlack); + else if (m_heap.objectSpace().hasImmortalBlocks() && Heap::isStartupSnapshotCell(cell)) [[unlikely]] + m_heap.willVisitSnapshotCell(const_cast(cell)); WTF::storeLoadFence(); diff --git a/Source/JavaScriptCore/heap/SlotVisitor.h b/Source/JavaScriptCore/heap/SlotVisitor.h index e34194d26666..943391cbb3b6 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.h +++ b/Source/JavaScriptCore/heap/SlotVisitor.h @@ -208,6 +208,9 @@ class SlotVisitor final : public AbstractSlotVisitor { void noteLiveAuxiliaryCell(HeapCell*); void visitChildren(const JSCell*); +#if USE(BUN_JSC_ADDITIONS) + void visitImmortalCellAsRoot(const JSCell*); +#endif void propagateExternalMemoryVisitedIfNecessary(); diff --git a/Source/JavaScriptCore/heap/WeakSet.h b/Source/JavaScriptCore/heap/WeakSet.h index 033fd4d45fca..95d6b2fb62b2 100644 --- a/Source/JavaScriptCore/heap/WeakSet.h +++ b/Source/JavaScriptCore/heap/WeakSet.h @@ -122,8 +122,12 @@ ALWAYS_INLINE void WeakSet::deallocate(WeakImpl* weakImpl) inline void WeakSet::resetAllocator() { - m_allocator = nullptr; - m_nextAllocator = m_blocks.head(); + // Compare first: this runs for every weak set on every full collection, including the snapshot cells' whose values are + // already these, and a store would dirty their pages for nothing. + if (m_allocator) + m_allocator = nullptr; + if (WeakBlock* head = m_blocks.head(); m_nextAllocator != head) + m_nextAllocator = head; } } // namespace JSC diff --git a/Source/JavaScriptCore/runtime/ButterflyInlines.h b/Source/JavaScriptCore/runtime/ButterflyInlines.h index d3f4333c2a4a..d3cbe750bea2 100644 --- a/Source/JavaScriptCore/runtime/ButterflyInlines.h +++ b/Source/JavaScriptCore/runtime/ButterflyInlines.h @@ -205,7 +205,9 @@ inline Butterfly* Butterfly::reallocArrayRightIfPossible( // We can eagerly destroy butterfly backed by PreciseAllocation if (1) concurrent collector is not active and (2) the butterfly does not contain any property storage. // This is because during deallocation concurrent collector can access butterfly and DFG concurrent compilers accesses properties. // Objects with no properties are common in arrays, and we are focusing on very large array crafted by repeating Array#push, so... that's fine! - bool canRealloc = !propertyCapacity && !vm.heap.mutatorShouldBeFenced() && std::bit_cast(theBase)->isPreciseAllocation(); + HeapCell* baseCell = std::bit_cast(theBase); + // A snapshot (immortal) allocation is left exactly as it is: growing it takes the allocate-and-copy path below instead. + bool canRealloc = !propertyCapacity && !vm.heap.mutatorShouldBeFenced() && baseCell->isPreciseAllocation() && !baseCell->preciseAllocation().isImmortal(); if (canRealloc) { void* newBase = vm.auxiliarySpace().reallocatePreciseAllocationNonVirtual(vm, std::bit_cast(theBase), newSize, &deferralContext, AllocationFailureMode::ReturnNull); if (!newBase) diff --git a/Source/JavaScriptCore/runtime/OptionsList.h b/Source/JavaScriptCore/runtime/OptionsList.h index 008d9452a9fc..53f95bc20fea 100644 --- a/Source/JavaScriptCore/runtime/OptionsList.h +++ b/Source/JavaScriptCore/runtime/OptionsList.h @@ -84,11 +84,14 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, useFFIDirectCall, true, Normal, "FTL calls the native FFI target directly (no invoke thunk)"_s) \ v(Bool, dumpFFIDisassembly, false, Normal, "disassemble generated FFI thunks/stubs"_s) \ v(Bool, verboseFFI, false, Normal, "dataLog on FFI thunk/stub/signature creation"_s) +#define FOR_EACH_JSC_STARTUP_SNAPSHOT_OPTION(v) \ + v(Bool, verboseStartupSnapshotFreeze, false, Normal, "log what freezing a heap into a snapshot settled and left behind"_s) #define FOR_EACH_JSC_CODEBLOCK_AGING_OPTION(v) \ v(Bool, useExecutionCountForCodeBlockAging, false, Normal, "If true, an LLInt/Baseline CodeBlock whose execution counter has advanced since the last old-age check is treated as still in use and its TTL is renewed instead of being jettisoned."_s) \ v(Double, codeBlockAgingLeaseMultiplier, 3.0, Normal, "When useExecutionCountForCodeBlockAging proves a CodeBlock is still active, renew its old-age TTL to this many multiples of timeToLive for its tier."_s) #else #define FOR_EACH_JSC_FFI_OPTION(v) +#define FOR_EACH_JSC_STARTUP_SNAPSHOT_OPTION(v) #define FOR_EACH_JSC_CODEBLOCK_AGING_OPTION(v) #endif @@ -111,6 +114,10 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, showPrivateScriptsInStackTraces, false, Normal, "Show private scripts in stack traces."_s) \ v(Bool, evalMode, false, Normal, "Set to true for less aggressive function call completion value discarding."_s) \ FOR_EACH_JSC_FFI_OPTION(v) \ + FOR_EACH_JSC_STARTUP_SNAPSHOT_OPTION(v) \ + v(Bool, useBorrowedBytecodeFromCache, true, Normal, "instruction streams decoded from a persistent (mmap'd/embedded) bytecode cache alias the cache instead of copying"_s) \ + v(Bool, useUnlinkedCodeBlockJettisoningForBytecodeCache, false, Normal, "If true, UnlinkedCodeBlocks decoded from a bytecode cache can also be jettisoned (regenerated from source or cache on next use)."_s) \ + v(Unsigned, unlinkedCodeBlockJettisonAge, 7, Normal, "Number of full collections an UnlinkedCodeBlock survives without being linked before it is jettisoned."_s) \ \ v(Unsigned, maxPerThreadStackUsage, 5 * MB, Normal, "Max allowed stack usage by the VM"_s) \ v(Unsigned, softReservedZoneSize, 128 * KB, Normal, "A buffer greater than reservedZoneSize that reserves space for stringifying exceptions."_s) \ diff --git a/Source/JavaScriptCore/runtime/Structure.cpp b/Source/JavaScriptCore/runtime/Structure.cpp index bd9e01f437e4..3633aac7d10d 100644 --- a/Source/JavaScriptCore/runtime/Structure.cpp +++ b/Source/JavaScriptCore/runtime/Structure.cpp @@ -44,6 +44,26 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { +#if USE(BUN_JSC_ADDITIONS) +// Striped by address; structures 32 bytes apart may share a stripe, which only costs contention. +static constexpr size_t snapshotSideLockStripes = 512; +static constexpr unsigned snapshotSideLockShift = 5; +static std::array s_snapshotSideLocks; + +ConcurrentJSLock& Structure::snapshotSideLock(const void* p) +{ + return s_snapshotSideLocks[(reinterpret_cast(p) >> snapshotSideLockShift) % snapshotSideLockStripes]; +} + +void Structure::prepareForStartupSnapshot(VM& vm) +{ + // Pinned: transitions clone the table instead of stealing it, so the snapshotd table is never taken away (a cell write). + if (ensurePropertyTableIfNotEmpty(vm)) + setIsPinnedPropertyTable(true); + setIsStartupSnapshotStructure(true); +} +#endif + template void Structure::checkOffsetConsistency(PropertyTable* propertyTable, const DetailsFunc& detailsFunc) const { @@ -436,7 +456,7 @@ bool Structure::findStructuresAndMapForMaterialization(Vector& st table = nullptr; for (structure = this; structure; structure = structure->previousID()) { - structure->m_lock.lock(); + structure->lock().lock(); table = structure->propertyTableOrNull(); if (table) { @@ -446,7 +466,7 @@ bool Structure::findStructuresAndMapForMaterialization(Vector& st } structures.append(structure); - structure->m_lock.unlock(); + structure->lock().unlock(); } ASSERT(!structure); @@ -471,14 +491,14 @@ PropertyTable* Structure::materializePropertyTable(VM& vm, bool setPropertyTable unsigned capacity = numberOfSlotsForMaxOffset(maxOffset(), m_inlineCapacity); if (didFindStructure) { table = table->copy(vm, capacity); - structure->m_lock.unlock(); + structure->lock().unlock(); } else table = PropertyTable::create(vm, capacity); // Must hold the lock on this structure, since we will be modifying this structure's // property map. We don't want getConcurrently() to see the property map in a half-baked // state. - GCSafeConcurrentJSLocker locker(m_lock, vm); + GCSafeConcurrentJSLocker locker(lock(), vm); if (setPropertyTable) this->setPropertyTable(vm, table); @@ -596,7 +616,7 @@ Structure* Structure::addNewPropertyTransition(VM& vm, Structure* structure, Pro // case all is well. If it wasn't for the lock, the GC would have TOCTOU: if could read // protectPropertyTableWhileTransitioning before we set it to true, and then blow the table away after. { - ConcurrentJSLocker locker(transition->m_lock); + ConcurrentJSLocker locker(transition->lock()); transition->setProtectPropertyTableWhileTransitioning(true); } @@ -617,7 +637,7 @@ Structure* Structure::addNewPropertyTransition(VM& vm, Structure* structure, Pro checkOffset(transition->transitionOffset(), transition->inlineCapacity()); if (!structure->hasBeenDictionary()) { - GCSafeConcurrentJSLocker locker(structure->m_lock, vm); + GCSafeConcurrentJSLocker locker(structure->lock(), vm); structure->m_transitionTable.add(vm, structure, transition); } transition->checkOffsetConsistency(); @@ -668,7 +688,7 @@ Structure* Structure::removePropertyTransitionFromExistingStructureConcurrently( unsigned attributes = 0; if (structure->getConcurrently(propertyName.uid(), attributes) == invalidOffset) return nullptr; - ConcurrentJSLocker locker(structure->m_lock); + ConcurrentJSLocker locker(structure->lock()); return removePropertyTransitionFromExistingStructureImpl(structure, propertyName, attributes, offset); } @@ -693,7 +713,7 @@ Structure* Structure::removeNewPropertyTransition(VM& vm, Structure* structure, // While we are deleting the property, we need to make sure the table is not cleared. { - ConcurrentJSLocker locker(transition->m_lock); + ConcurrentJSLocker locker(transition->lock()); transition->setProtectPropertyTableWhileTransitioning(true); } @@ -714,7 +734,7 @@ Structure* Structure::removeNewPropertyTransition(VM& vm, Structure* structure, checkOffset(transition->transitionOffset(), transition->inlineCapacity()); if (!structure->hasBeenDictionary()) { - GCSafeConcurrentJSLocker locker(structure->m_lock, vm); + GCSafeConcurrentJSLocker locker(structure->lock(), vm); structure->m_transitionTable.add(vm, structure, transition); } transition->checkOffsetConsistency(); @@ -744,13 +764,13 @@ Structure* Structure::changePrototypeTransition(VM& vm, Structure* structure, JS // Let's pin the table and break the edge to the previous Structure. Structure* transition = Structure::create(vm, structure, &deferred); PropertyTable* table = structure->copyPropertyTableForPinning(vm); - transition->pin(Locker { transition->m_lock }, vm, table); + transition->pin(Locker { transition->lock() }, vm, table); transition->m_prototype.set(vm, transition, prototype); transition->setTransitionKind(TransitionKind::ChangePrototype); transition->setMaxOffset(vm, structure->maxOffset()); checkOffset(transition->transitionOffset(), transition->inlineCapacity()); if (shouldChain) { - GCSafeConcurrentJSLocker locker(structure->m_lock, vm); + GCSafeConcurrentJSLocker locker(structure->lock(), vm); structure->m_transitionTable.add(vm, structure, transition); } @@ -767,7 +787,7 @@ Structure* Structure::changeGlobalProxyTargetTransition(VM& vm, Structure* struc transition->setRealm(vm, globalObject); PropertyTable* table = structure->copyPropertyTableForPinning(vm); - transition->pin(Locker { transition->m_lock }, vm, table); + transition->pin(Locker { transition->lock() }, vm, table); transition->setMaxOffset(vm, structure->maxOffset()); transition->checkOffsetConsistency(); @@ -800,7 +820,7 @@ Structure* Structure::attributeChangeTransitionToExistingStructure(Structure* st Structure* Structure::attributeChangeTransitionToExistingStructureConcurrently(Structure* structure, PropertyName propertyName, unsigned attributes, PropertyOffset& offset) { - ConcurrentJSLocker locker(structure->m_lock); + ConcurrentJSLocker locker(structure->lock()); return attributeChangeTransitionToExistingStructureImpl(structure, propertyName, attributes, offset); } @@ -834,7 +854,7 @@ Structure* Structure::attributeChangeTransition(VM& vm, Structure* structure, Pr transition->m_cachedPrototypeChain.setMayBeNull(vm, transition, structure->m_cachedPrototypeChain.get()); { - ConcurrentJSLocker locker(transition->m_lock); + ConcurrentJSLocker locker(transition->lock()); transition->setProtectPropertyTableWhileTransitioning(true); } @@ -855,7 +875,7 @@ Structure* Structure::attributeChangeTransition(VM& vm, Structure* structure, Pr checkOffset(transition->transitionOffset(), transition->inlineCapacity()); if (!structure->hasBeenDictionary()) { - GCSafeConcurrentJSLocker locker(structure->m_lock, vm); + GCSafeConcurrentJSLocker locker(structure->lock(), vm); structure->m_transitionTable.add(vm, structure, transition); } transition->checkOffsetConsistency(); @@ -871,7 +891,7 @@ Structure* Structure::toDictionaryTransition(VM& vm, Structure* structure, Dicti Structure* transition = Structure::create(vm, structure, deferred); PropertyTable* table = structure->copyPropertyTableForPinning(vm); - transition->pin(Locker { transition->m_lock }, vm, table); + transition->pin(Locker { transition->lock() }, vm, table); transition->setMaxOffset(vm, structure->maxOffset()); transition->setDictionaryKind(kind); transition->setHasBeenDictionary(true); @@ -917,7 +937,7 @@ PropertyTable* Structure::takePropertyTableOrCloneIfPinned(VM& vm) if (result) { if (isPinnedPropertyTable()) return result->copy(vm, result->size() + 1); - ConcurrentJSLocker locker(m_lock); + ConcurrentJSLocker locker(lock()); setPropertyTable(vm, nullptr); return result; } @@ -963,7 +983,7 @@ Structure* Structure::nonPropertyTransitionSlow(VM& vm, Structure* structure, Tr ASSERT(transitionKind == TransitionKind::Seal || transitionKind == TransitionKind::Freeze); PropertyTable* table = structure->copyPropertyTableForPinning(vm); - transition->pinForCaching(Locker { transition->m_lock }, vm, table); + transition->pinForCaching(Locker { transition->lock() }, vm, table); transition->setMaxOffset(vm, structure->maxOffset()); table = transition->propertyTableOrNull(); @@ -988,9 +1008,9 @@ Structure* Structure::nonPropertyTransitionSlow(VM& vm, Structure* structure, Tr if (structure->isDictionary()) { PropertyTable* table = transition->ensurePropertyTable(vm); - transition->pin(Locker { transition->m_lock }, vm, table); + transition->pin(Locker { transition->lock() }, vm, table); } else { - Locker locker { structure->m_lock }; + Locker locker { structure->lock() }; structure->m_transitionTable.add(vm, structure, transition); } @@ -1047,7 +1067,7 @@ Structure* Structure::flattenDictionaryStructure(VM& vm, JSObject* object) if (beforeOutOfLineCapacity != afterOutOfLineCapacity) cellLocker = Locker { object->cellLock() }; - GCSafeConcurrentJSLocker locker(m_lock, vm); + GCSafeConcurrentJSLocker locker(lock(), vm); object->setStructureIDDirectly(id().nuke()); WTF::storeStoreFence(); @@ -1136,7 +1156,7 @@ WatchpointSet* Structure::ensurePropertyReplacementWatchpointSet(VM& vm, Propert if (!hasRareData()) allocateRareData(vm); - ConcurrentJSLocker locker(m_lock); + ConcurrentJSLocker locker(lock()); Structure* structure = this; StructureRareData* rareData = structure->rareData(); auto result = rareData->m_replacementWatchpointSets.add(offset, nullptr); @@ -1243,8 +1263,8 @@ PropertyOffset Structure::getConcurrently(UniquedStringImpl* uid, unsigned& attr case TransitionKind::PropertyDeletion: if (structure->m_transitionPropertyName.get() == uid) { if (didFindStructure) { - assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. - tableStructure->m_lock.unlock(); + assertIsHeld(tableStructure->lock()); // Sadly Clang needs some help here. + tableStructure->lock().unlock(); } return invalidOffset; } @@ -1260,8 +1280,8 @@ PropertyOffset Structure::getConcurrently(UniquedStringImpl* uid, unsigned& attr PropertyOffset result = structure->transitionOffset(); attributes = structure->transitionPropertyAttributes(); if (didFindStructure) { - assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. - tableStructure->m_lock.unlock(); + assertIsHeld(tableStructure->lock()); // Sadly Clang needs some help here. + tableStructure->lock().unlock(); } return result; } @@ -1270,7 +1290,7 @@ PropertyOffset Structure::getConcurrently(UniquedStringImpl* uid, unsigned& attr PropertyOffset result = invalidOffset; if (didFindStructure) { - assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. + assertIsHeld(tableStructure->lock()); // Sadly Clang needs some help here. // Because uid is UniquedStringImpl, it is guaranteed that the hash is already computed. // So we can use PropertyTable::get even from the concurrent compilers. // Even though taking a lock, all you can do is getting value from this table. We must not modify the table @@ -1280,7 +1300,7 @@ PropertyOffset Structure::getConcurrently(UniquedStringImpl* uid, unsigned& attr result = offset; attributes = entryAttributes; } - tableStructure->m_lock.unlock(); + tableStructure->lock().unlock(); } return result; @@ -1407,13 +1427,18 @@ void Structure::visitChildrenImpl(JSCell* cell, Visitor& visitor) Base::visitChildren(thisObject, visitor); - ConcurrentJSLocker locker(thisObject->m_lock); + // Snapshot structures: visit without taking lock() (an identity write) and without dropping caches, so their pages stay clean. + bool isSnapshot = thisObject->isStartupSnapshotStructure(); + std::optional locker; + if (!isSnapshot) + locker.emplace(thisObject->lock()); visitor.append(thisObject->m_realm); if (!thisObject->isObject()) { // We do not need to clear JSPropertyNameEnumerator since it is never cached for non-object Structure. // We do not have code clearing JSPropertyNameEnumerator since this function can be called concurrently. - thisObject->m_cachedPrototypeChain.clear(); + if (!isSnapshot || thisObject->m_cachedPrototypeChain) + thisObject->m_cachedPrototypeChain.clear(); #if ASSERT_ENABLED if (auto* rareData = thisObject->tryRareData()) ASSERT(!rareData->cachedPropertyNameEnumerator()); @@ -1424,7 +1449,7 @@ void Structure::visitChildrenImpl(JSCell* cell, Visitor& visitor) } visitor.append(thisObject->m_previousOrRareData); - if (thisObject->isPinnedPropertyTable() || thisObject->protectPropertyTableWhileTransitioning()) { + if (isSnapshot || thisObject->isPinnedPropertyTable() || thisObject->protectPropertyTableWhileTransitioning()) { // NOTE: This can interleave in pin(), in which case it may see a null property table. // That's fine, because then the barrier will fire and we will scan this again. visitor.append(thisObject->m_propertyTableUnsafe); @@ -1703,7 +1728,7 @@ Structure* Structure::setBrandTransitionFromExistingStructureImpl(Structure* str Structure* Structure::setBrandTransitionFromExistingStructureConcurrently(Structure* structure, UniquedStringImpl* brandID) { - ConcurrentJSLocker locker(structure->m_lock); + ConcurrentJSLocker locker(structure->lock()); return setBrandTransitionFromExistingStructureImpl(structure, brandID); } @@ -1726,9 +1751,9 @@ Structure* Structure::setBrandTransition(VM& vm, Structure* structure, Symbol* b if (structure->isDictionary()) { PropertyTable* table = transition->ensurePropertyTable(vm); - transition->pin(Locker { transition->m_lock }, vm, table); + transition->pin(Locker { transition->lock() }, vm, table); } else { - Locker locker { structure->m_lock }; + Locker locker { structure->lock() }; structure->m_transitionTable.add(vm, structure, transition); } @@ -1825,6 +1850,7 @@ void Structure::checkConsistency() } #endif + } // namespace JSC WTF_ALLOW_UNSAFE_BUFFER_USAGE_END diff --git a/Source/JavaScriptCore/runtime/Structure.h b/Source/JavaScriptCore/runtime/Structure.h index edeefa4f59da..bd82f2e13131 100644 --- a/Source/JavaScriptCore/runtime/Structure.h +++ b/Source/JavaScriptCore/runtime/Structure.h @@ -782,7 +782,15 @@ class Structure : public JSCell { static void dumpContextHeader(PrintStream&); +#if USE(BUN_JSC_ADDITIONS) + // Snapshot structures are locked through a striped side lock, so taking the lock never dirties their (shared, file-backed) pages. + ConcurrentJSLock& lock() LIFETIME_BOUND { if (isStartupSnapshotStructure()) [[unlikely]] return snapshotSideLock(this); return m_lock; } + JS_EXPORT_PRIVATE void prepareForStartupSnapshot(VM&); // heap freeze: settle what would otherwise be written into the cell later + bool hasPropertyTableForSnapshot() const { return !!m_propertyTableUnsafe.get(); } + JS_EXPORT_PRIVATE static ConcurrentJSLock& snapshotSideLock(const void*); +#else ConcurrentJSLock& lock() LIFETIME_BOUND { return m_lock; } +#endif unsigned propertyHash() const { return m_propertyHash; } SeenProperties seenProperties() const { return m_seenProperties; } @@ -850,6 +858,11 @@ class Structure : public JSCell { DEFINE_BITFIELD(bool, hasUnderscoreProtoPropertyExcludingOriginalProto, HasUnderscoreProtoPropertyExcludingOriginalProto, 1, 28); DEFINE_BITFIELD(bool, hasNonConfigurableProperties, HasNonConfigurableProperties, 1, 29); DEFINE_BITFIELD(bool, hasNonConfigurableReadOnlyOrGetterSetterProperties, HasNonConfigurableReadOnlyOrGetterSetterProperties, 1, 30); +#if USE(BUN_JSC_ADDITIONS) + DEFINE_BITFIELD(bool, isStartupSnapshotStructure, IsStartupSnapshotStructure, 1, 31); +#else + constexpr bool isStartupSnapshotStructure() const { return false; } +#endif enum class StructureVariant : uint8_t { Normal, @@ -915,7 +928,7 @@ class Structure : public JSCell { // and the list of structures that we visited before we got to it. If it returns a // non-null structure, it will also lock the structure that it returns; it is your job // to unlock it. - bool findStructuresAndMapForMaterialization(Vector& structures, Structure*& structure, PropertyTable*&) WTF_ACQUIRES_LOCK_IF(true, structure->m_lock); + bool findStructuresAndMapForMaterialization(Vector& structures, Structure*& structure, PropertyTable*&) WTF_ACQUIRES_LOCK_IF(true, structure->lock()); static Structure* toDictionaryTransition(VM&, Structure*, DictionaryKind, DeferredStructureTransitionWatchpointFire* = nullptr); diff --git a/Source/JavaScriptCore/runtime/StructureInlines.h b/Source/JavaScriptCore/runtime/StructureInlines.h index 8ad15bb635ae..4d3994311563 100644 --- a/Source/JavaScriptCore/runtime/StructureInlines.h +++ b/Source/JavaScriptCore/runtime/StructureInlines.h @@ -102,15 +102,15 @@ void Structure::forEachPropertyConcurrently(const Functor& functor) if (!functor(PropertyTableEntry(structure->m_transitionPropertyName.get(), structure->transitionOffset(), structure->transitionPropertyAttributes()))) { if (didFindStructure) { - assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. - tableStructure->m_lock.unlock(); + assertIsHeld(tableStructure->lock()); // Sadly Clang needs some help here. + tableStructure->lock().unlock(); } return; } } if (didFindStructure) { - assertIsHeld(tableStructure->m_lock); // Sadly Clang needs some help here. + assertIsHeld(tableStructure->lock()); // Sadly Clang needs some help here. table->forEachProperty([&](const auto& entry) { if (seenProperties.contains(entry.key())) return IterationStatus::Continue; @@ -120,7 +120,7 @@ void Structure::forEachPropertyConcurrently(const Functor& functor) return IterationStatus::Continue; }); - tableStructure->m_lock.unlock(); + tableStructure->lock().unlock(); } } @@ -208,7 +208,7 @@ inline void Structure::didCachePropertyReplacement(VM& vm, PropertyOffset offset inline WatchpointSet* Structure::propertyReplacementWatchpointSet(PropertyOffset offset) { - ConcurrentJSLocker locker(m_lock); + ConcurrentJSLocker locker(lock()); StructureRareData* rareData = tryRareData(); if (!rareData) return nullptr; @@ -237,7 +237,7 @@ inline PropertyOffset Structure::add(VM& vm, PropertyName propertyName, unsigned ASSERT(!isCompilationThread()); PropertyTable* table = ensurePropertyTable(vm); - GCSafeConcurrentJSLocker locker(m_lock, vm); + GCSafeConcurrentJSLocker locker(lock(), vm); switch (shouldPin) { case ShouldPin::Yes: @@ -293,7 +293,7 @@ inline PropertyOffset Structure::remove(VM& vm, PropertyName propertyName, const { ASSERT(!isCompilationThread()); PropertyTable* table = ensurePropertyTable(vm); - GCSafeConcurrentJSLocker locker(m_lock, vm); + GCSafeConcurrentJSLocker locker(lock(), vm); switch (shouldPin) { case ShouldPin::Yes: @@ -336,7 +336,7 @@ inline PropertyOffset Structure::attributeChange(VM& vm, PropertyName propertyNa ASSERT(!isCompilationThread()); PropertyTable* table = ensurePropertyTable(vm); - GCSafeConcurrentJSLocker locker(m_lock, vm); + GCSafeConcurrentJSLocker locker(lock(), vm); switch (shouldPin) { case ShouldPin::Yes: @@ -404,7 +404,7 @@ ALWAYS_INLINE auto Structure::addOrReplacePropertyWithoutTransition(VM& vm, Prop if (findResult.offset != invalidOffset) return std::tuple { findResult.offset, findResult.attributes, false }; - GCSafeConcurrentJSLocker locker(m_lock, vm); + GCSafeConcurrentJSLocker locker(lock(), vm); pin(locker, vm, table); @@ -575,7 +575,7 @@ ALWAYS_INLINE Structure* Structure::addPropertyTransitionToExistingStructure(Str ALWAYS_INLINE Structure* Structure::addPropertyTransitionToExistingStructureConcurrently(Structure* structure, UniquedStringImpl* uid, unsigned attributes, PropertyOffset& offset) { - ConcurrentJSLocker locker(structure->m_lock); + ConcurrentJSLocker locker(structure->lock()); return addPropertyTransitionToExistingStructureImpl(structure, uid, attributes, offset); } diff --git a/Source/WTF/wtf/text/StringImpl.h b/Source/WTF/wtf/text/StringImpl.h index 905fd5318c1a..35ac6a6b1f97 100644 --- a/Source/WTF/wtf/text/StringImpl.h +++ b/Source/WTF/wtf/text/StringImpl.h @@ -328,6 +328,8 @@ class StringImpl : private StringImplShape, public NoVirtualDestructorBase { size_t cost() const; size_t costDuringGC(); + // Snapshot: do the lazy one-time writes (hash, did-report-cost) now so the string's header never changes after the freeze. + void settleLazyHeaderWritesForStartupSnapshot() const { hash(); m_hashAndFlags |= s_hashFlagDidReportCost; } WTF_EXPORT_PRIVATE size_t NODELETE sizeInBytes() const; @@ -381,6 +383,7 @@ class StringImpl : private StringImplShape, public NoVirtualDestructorBase { unsigned existingSymbolAwareHash() const; SUPPRESS_TSAN bool isStatic() const { return m_refCount.load(std::memory_order_relaxed) & s_refCountFlagIsStaticString; } + void makeStaticForSnapshot() { m_refCount.fetch_or(s_refCountFlagIsStaticString, std::memory_order_relaxed); } uint32_t refCount() const { return m_refCount.load(std::memory_order_relaxed) / s_refCountIncrement; } bool hasOneRef() const { return m_refCount.load(std::memory_order_relaxed) == s_refCountIncrement; } @@ -1184,10 +1187,9 @@ inline void StringImpl::ref() { STRING_STATS_REF_STRING(*this); -#if TSAN_ENABLED + // Static strings (which every string in a snapshot is made) never write their refcount, so their pages stay clean. if (isStatic()) return; -#endif m_refCount.fetch_add(s_refCountIncrement, std::memory_order_relaxed); } @@ -1196,10 +1198,8 @@ inline void StringImpl::deref() { STRING_STATS_DEREF_STRING(*this); -#if TSAN_ENABLED if (isStatic()) return; -#endif auto oldRefCount = m_refCount.fetch_sub(s_refCountIncrement, std::memory_order_relaxed); if (oldRefCount != s_refCountIncrement) From 42f58558054f783826566290e8bb83833a6e7d8a Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 10 Aug 2026 06:05:13 -0700 Subject: [PATCH 2/4] Bytecode cache: alias persistent payloads instead of copying them; allow re-decoding jettisoned code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a CachedBytecode payload is persistent (mapped from a file or embedded in the executable), the instruction streams and expression info decoded from it borrow the payload's bytes instead of copying them (useBorrowedBytecodeFromCache). InstructionStream refs read through a span owned by the stream — the writer refreshes it after every mutation — so owned and borrowed streams share one access path. UnlinkedFunctionExecutables generated from a cache remember where their record lives, so a code block that was jettisoned (useUnlinkedCodeBlockJettisoningForBytecodeCache) can be decoded again on demand; for executables that are part of a snapshot the decoded block goes to the heap's side table rather than into the never-written cell. --- .../bytecode/BytecodeRewriter.cpp | 1 + .../bytecode/ExpressionInfo.cpp | 10 ++ .../JavaScriptCore/bytecode/ExpressionInfo.h | 7 +- .../bytecode/InstructionStream.h | 160 ++++++++++++------ .../bytecode/UnlinkedFunctionExecutable.cpp | 66 +++++++- .../bytecode/UnlinkedFunctionExecutable.h | 18 +- Source/JavaScriptCore/runtime/CachePayload.h | 4 + .../JavaScriptCore/runtime/CachedBytecode.h | 2 + Source/JavaScriptCore/runtime/CachedTypes.cpp | 92 ++++++++-- Source/JavaScriptCore/runtime/CachedTypes.h | 14 +- 10 files changed, 295 insertions(+), 79 deletions(-) diff --git a/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp b/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp index 932fbc882153..317f322a3e75 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp +++ b/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp @@ -49,6 +49,7 @@ void BytecodeRewriter::applyModification() m_writer.m_instructions.insertVector(insertion.index.bytecodeOffset, insertion.instructions.m_instructions); } } + m_writer.didMutateBuffer(); m_insertions.clear(); } diff --git a/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp b/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp index 8189d7c39efe..1819129a231b 100644 --- a/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp +++ b/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp @@ -898,6 +898,14 @@ std::unique_ptr ExpressionInfo::createUninitialized(unsigned num return std::unique_ptr(new (allocation) ExpressionInfo(numberOfChapters, numberOfEncodedInfo, numberOfEncodedInfoExtensions)); } +std::unique_ptr ExpressionInfo::createBorrowed(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions, const unsigned* payload) +{ + void* allocation = FastMalloc::malloc(sizeof(ExpressionInfo)); + auto info = std::unique_ptr(new (allocation) ExpressionInfo(numberOfChapters, numberOfEncodedInfo, numberOfEncodedInfoExtensions)); + info->m_borrowedPayload = const_cast(payload); + return info; +} + ExpressionInfo::ExpressionInfo(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions) : m_numberOfChapters(numberOfChapters) , m_numberOfEncodedInfo(numberOfEncodedInfo) @@ -914,6 +922,8 @@ ExpressionInfo::ExpressionInfo(Vector&& chapters, Vector&& size_t ExpressionInfo::byteSize() const { + if (m_borrowedPayload) [[unlikely]] + return sizeof(ExpressionInfo); // the payload belongs to the cache mapping, not to this object return totalSizeInBytes(m_numberOfChapters, m_numberOfEncodedInfo, m_numberOfEncodedInfoExtensions); } diff --git a/Source/JavaScriptCore/bytecode/ExpressionInfo.h b/Source/JavaScriptCore/bytecode/ExpressionInfo.h index 8a79926a83de..d2935b87d8be 100644 --- a/Source/JavaScriptCore/bytecode/ExpressionInfo.h +++ b/Source/JavaScriptCore/bytecode/ExpressionInfo.h @@ -214,7 +214,7 @@ class ExpressionInfo { Chapter* chapters() const { - return std::bit_cast(this + 1); + return std::bit_cast(payload()); } EncodedInfo* encodedInfo() const @@ -239,10 +239,14 @@ class ExpressionInfo { unsigned* payload() const { + if (m_borrowedPayload) [[unlikely]] + return m_borrowedPayload; return std::bit_cast(this + 1); } static std::unique_ptr createUninitialized(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions); + // Header-only object whose (immutable) payload lives elsewhere, e.g. inside an mmap'd bytecode cache. + static std::unique_ptr createBorrowed(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions, const unsigned* payload); static constexpr unsigned bitsPerWord = sizeof(unsigned) * CHAR_BIT; @@ -323,6 +327,7 @@ class ExpressionInfo { unsigned m_numberOfChapters; unsigned m_numberOfEncodedInfo; unsigned m_numberOfEncodedInfoExtensions; + unsigned* m_borrowedPayload { nullptr }; // Followed by the following which are allocated but are dynamically sized. // Chapter chapters[numberOfChapters]; // EncodedInfo encodedInfo[numberOfEncodedInfo + numberOfEncodedInfoExtensions]; diff --git a/Source/JavaScriptCore/bytecode/InstructionStream.h b/Source/JavaScriptCore/bytecode/InstructionStream.h index e094a85db136..f53b1ba85c1e 100644 --- a/Source/JavaScriptCore/bytecode/InstructionStream.h +++ b/Source/JavaScriptCore/bytecode/InstructionStream.h @@ -30,6 +30,7 @@ #include "Instruction.h" #include #include +#include WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -52,89 +53,90 @@ class InstructionStream { size_t sizeInBytes() const { - return m_instructions.size(); + return m_bytes.size(); } + size_t ownedSizeInBytes() const { return m_isBorrowed ? 0 : m_bytes.size(); } // what this object reports to the GC as its own using Offset = unsigned; private: - template + // Refs read through the stream's span (one indirection, exactly like reading through a reference to the buffer), so a + // ref minted while a writer is still appending keeps seeing the current bytes: the writer refreshes the span after + // every mutation of its buffer. + using Bytes = std::span; + +public: + class Ref; +private: class BaseRef { WTF_DEPRECATED_MAKE_FAST_ALLOCATED(BaseRef); template friend class InstructionStream; + template friend class InstructionStreamWriter; public: - BaseRef(const BaseRef& other) - : m_instructions(other.m_instructions) - , m_index(other.m_index) - { } - - void operator=(const BaseRef& other) - { - m_instructions = other.m_instructions; - m_index = other.m_index; - } - inline const InstructionType* operator->() const { return unwrap(); } inline const InstructionType* ptr() const { return unwrap(); } - bool operator==(const BaseRef& other) const + bool operator==(const BaseRef& other) const { - return &m_instructions == &other.m_instructions && m_index == other.m_index; + return m_bytes == other.m_bytes && m_index == other.m_index; } - BaseRef next() const - { - return BaseRef { m_instructions, m_index + ptr()->size() }; - } + inline Ref next() const; inline Offset offset() const { return m_index; } inline BytecodeIndex index() const { return BytecodeIndex(offset()); } bool isValid() const { - return m_index < m_instructions.size(); + return m_index < m_bytes->size(); } - private: - inline const InstructionType* unwrap() const { return reinterpret_cast(&m_instructions[m_index]); } - protected: - BaseRef(InstructionBuffer& instructions, size_t index) - : m_instructions(instructions) + BaseRef(const Bytes* bytes, size_t index) + : m_bytes(bytes) , m_index(index) { } - InstructionBuffer& m_instructions; + inline const InstructionType* unwrap() const { return reinterpret_cast(m_bytes->data() + m_index); } + + const Bytes* m_bytes; Offset m_index; }; public: - using Ref = BaseRef; + class Ref : public BaseRef { + template friend class InstructionStream; + template friend class InstructionStreamWriter; + friend class BaseRef; + protected: + using BaseRef::BaseRef; + }; - class MutableRef : public BaseRef { + class MutableRef : public BaseRef { + template friend class InstructionStream; template friend class InstructionStreamWriter; protected: - using BaseRef::BaseRef; - using BaseRef::m_index; - using BaseRef::m_instructions; + MutableRef(InstructionBuffer& buffer, const Bytes* bytes, size_t index) + : BaseRef(bytes, index) + , m_buffer(&buffer) + { } + using BaseRef::m_index; public: - Ref freeze() const { return Ref { m_instructions, m_index }; } + Ref freeze() const { return Ref { this->m_bytes, m_index }; } inline InstructionType* operator->() { return unwrap(); } - inline const InstructionType* operator->() const { return unwrap(); } + inline const InstructionType* operator->() const { return BaseRef::unwrap(); } inline InstructionType* ptr() { return unwrap(); } - inline const InstructionType* ptr() const { return unwrap(); } - inline operator Ref() - { - return Ref { m_instructions, m_index }; - } + inline const InstructionType* ptr() const { return BaseRef::unwrap(); } + inline operator Ref() const { return freeze(); } private: - inline InstructionType* unwrap() { return reinterpret_cast(&m_instructions[m_index]); } - inline const InstructionType* unwrap() const { return reinterpret_cast(&m_instructions[m_index]); } + inline InstructionType* unwrap() { return reinterpret_cast(m_buffer->mutableSpan().data() + m_index); } + + InstructionBuffer* m_buffer; }; private: @@ -165,45 +167,86 @@ class InstructionStream { public: inline iterator begin() const LIFETIME_BOUND { - return iterator { m_instructions, 0 }; + return iterator { &m_bytes, 0 }; } inline iterator end() const LIFETIME_BOUND { - return iterator { m_instructions, m_instructions.size() }; + return iterator { &m_bytes, m_bytes.size() }; } inline const Ref at(BytecodeIndex index) const { return at(index.offset()); } inline const Ref at(Offset offset) const { - ASSERT(offset < m_instructions.size()); - return Ref { m_instructions, offset }; + ASSERT(offset < m_bytes.size()); + return Ref { &m_bytes, offset }; } inline size_t size() const { - return m_instructions.size(); + return m_bytes.size(); } const void* rawPointer() const { - return m_instructions.span().data(); + return m_bytes.data(); } bool contains(InstructionType* instruction) const { auto* pointer = std::bit_cast(instruction); - return pointer >= m_instructions.begin() && pointer < m_instructions.end(); + return pointer >= m_bytes.data() && pointer < m_bytes.data() + m_bytes.size(); + } + + // Read an immutable instruction stream that lives elsewhere (inside a bytecode cache mapping) instead of copying it. + enum BorrowTag { Borrow }; + InstructionStream(Bytes borrowed, BorrowTag) + : m_bytes(borrowed) + , m_isBorrowed(true) + { } + bool isBorrowed() const { return m_isBorrowed; } + + InstructionStream(InstructionStream&& other) + : m_instructions(WTF::move(other.m_instructions)) + , m_bytes(other.m_isBorrowed ? other.m_bytes : Bytes(m_instructions.span())) + , m_isBorrowed(other.m_isBorrowed) + { + other.m_bytes = { }; + other.m_isBorrowed = false; + } + InstructionStream& operator=(InstructionStream&& other) + { + m_instructions = WTF::move(other.m_instructions); + m_isBorrowed = other.m_isBorrowed; + m_bytes = m_isBorrowed ? other.m_bytes : Bytes(m_instructions.span()); + other.m_bytes = { }; + other.m_isBorrowed = false; + return *this; } protected: explicit InstructionStream(InstructionBuffer&& instructions) : m_instructions(WTF::move(instructions)) + , m_bytes(m_instructions.span()) { } + void didMutateBuffer() + { + RELEASE_ASSERT(!m_isBorrowed); // the bytes are the cache mapping's; every write path comes through here + m_bytes = m_instructions.span(); + } + InstructionBuffer m_instructions; + Bytes m_bytes; + bool m_isBorrowed { false }; }; +template +inline typename InstructionStream::Ref InstructionStream::BaseRef::next() const +{ + return Ref { m_bytes, m_index + ptr()->size() }; +} + template class InstructionStreamWriter : public InstructionStream { friend class BytecodeRewriter; @@ -213,6 +256,8 @@ class InstructionStreamWriter : public InstructionStream { using typename InstructionStream::MutableRef; using typename InstructionStream::Offset; using InstructionStream::m_instructions; + using InstructionStream::m_bytes; + using InstructionStream::didMutateBuffer; InstructionStreamWriter() : InstructionStream({ }) @@ -223,12 +268,13 @@ class InstructionStreamWriter : public InstructionStream { RELEASE_ASSERT(!m_instructions.size()); RELEASE_ASSERT(!buffer.size()); m_instructions = WTF::move(buffer); + didMutateBuffer(); } inline MutableRef ref(Offset offset) { ASSERT(offset < m_instructions.size()); - return MutableRef { m_instructions, offset }; + return MutableRef { m_instructions, &m_bytes, offset }; } void seek(unsigned position) @@ -255,8 +301,10 @@ class InstructionStreamWriter : public InstructionStream { uint8_t* reserve() { ASSERT(!m_finalized); - if ((m_position + size) > m_instructions.size()) + if ((m_position + size) > m_instructions.size()) { m_instructions.grow(m_position + size); + didMutateBuffer(); + } auto* result = m_instructions.mutableSpan().data() + m_position; m_position += size; return result; @@ -266,6 +314,7 @@ class InstructionStreamWriter : public InstructionStream { { ASSERT(ref.offset() < m_instructions.size()); m_instructions.shrink(ref.offset()); + didMutateBuffer(); m_position = ref.offset(); } @@ -273,7 +322,9 @@ class InstructionStreamWriter : public InstructionStream { { m_finalized = true; m_instructions.shrinkToFit(); - return std::unique_ptr> { new InstructionStream(WTF::move(m_instructions)) }; + auto result = std::unique_ptr> { new InstructionStream(WTF::move(m_instructions)) }; + didMutateBuffer(); + return result; } std::unique_ptr> finalize(InstructionBuffer& usedBuffer) @@ -285,13 +336,14 @@ class InstructionStreamWriter : public InstructionStream { memcpy(resultBuffer.mutableSpan().data(), m_instructions.span().data(), m_instructions.sizeInBytes()); usedBuffer = WTF::move(m_instructions); + didMutateBuffer(); return std::unique_ptr> { new InstructionStream(WTF::move(resultBuffer)) }; } MutableRef ref() { - return MutableRef { m_instructions, m_position }; + return MutableRef { m_instructions, &m_bytes, m_position }; } void swap(InstructionStreamWriter& other) @@ -299,6 +351,8 @@ class InstructionStreamWriter : public InstructionStream { std::swap(m_finalized, other.m_finalized); std::swap(m_position, other.m_position); m_instructions.swap(other.m_instructions); + didMutateBuffer(); + other.didMutateBuffer(); } private: @@ -330,12 +384,12 @@ class InstructionStreamWriter : public InstructionStream { public: iterator begin() { - return iterator { m_instructions, 0 }; + return iterator { m_instructions, &m_bytes, 0 }; } iterator end() { - return iterator { m_instructions, m_instructions.size() }; + return iterator { m_instructions, &m_bytes, m_instructions.size() }; } private: diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp index 7f5960c0ce39..ee483638cc2f 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp @@ -169,7 +169,7 @@ void UnlinkedFunctionExecutable::visitChildrenImpl(JSCell* cell, Visitor& visito return; if (unlinkedCodeBlock->didOptimize() == TriState::True) visitor.append(unlinkedCodeBlock); - else if (unlinkedCodeBlock->age() < UnlinkedCodeBlock::maxAge) + else if (unlinkedCodeBlock->age() < std::min(Options::unlinkedCodeBlockJettisonAge(), UnlinkedCodeBlock::maxAge)) visitor.append(unlinkedCodeBlock); }; markIfProfitable(thisObject->m_unlinkedCodeBlockForCall); @@ -239,6 +239,37 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind, OptionSet codeGenerationMode, ParserError& error, SourceParseMode parseMode) { +#if USE(BUN_JSC_ADDITIONS) + if (vm.heap.objectSpace().hasImmortalBlocks() && Heap::isStartupSnapshotCell(this)) [[unlikely]] { + // Snapshot executable: its cell is never written. Whatever it already links to (snapshotd) is used as is; anything decoded/generated now lives in the heap's side table. + if (!m_isCached) { + if (auto* codeBlock = (specializationKind == CodeSpecializationKind::CodeForCall ? m_unlinkedCodeBlockForCall : m_unlinkedCodeBlockForConstruct).get()) + return codeBlock; + } + if (auto* codeBlock = vm.heap.snapshotUnlinkedCodeBlockFor(this, specializationKind)) + return codeBlock; + WriteBarrier slot; + DeferGC deferGC(vm); + if (m_isCached) { + int32_t offset = specializationKind == CodeSpecializationKind::CodeForCall ? m_cachedCodeBlockForCallOffset : m_cachedCodeBlockForConstructOffset; + if (offset && m_decoder) { + // m_decoder is the build process's, frozen with this cell: only read from it. The decode goes through this + // process's own decoder for the same payload, and records nothing (this code was recorded when first decoded). + Ref decoder = vm.ensureBytecodeCacheDecoder(Ref { m_decoder->cachedBytecode() }, m_decoder->provider()); + decodeFunctionCodeBlockForReDecode(decoder.get(), offset, slot, this); + } + } else + decodeCodeBlockFromCacheRecord(vm, source, specializationKind, slot); + if (!slot) { + UnlinkedFunctionCodeBlock* result = generateUnlinkedFunctionCodeBlock(vm, this, source, specializationKind, codeGenerationMode, isBuiltinFunction() ? UnlinkedBuiltinFunction : UnlinkedNormalFunction, error, parseMode); + if (error.isValid()) + return nullptr; + slot.setWithoutWriteBarrier(result); + } + vm.heap.setSnapshotUnlinkedCodeBlockFor(this, specializationKind, slot.get()); + return slot.get(); + } +#endif if (m_isCached) decodeCachedCodeBlocks(vm); switch (specializationKind) { @@ -252,6 +283,9 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( break; } + if (UnlinkedFunctionCodeBlock* redecoded = tryRedecodeCodeBlock(vm, source, specializationKind)) + return redecoded; + UnlinkedFunctionCodeBlock* result = generateUnlinkedFunctionCodeBlock( vm, this, source, specializationKind, codeGenerationMode, isBuiltinFunction() ? UnlinkedBuiltinFunction : UnlinkedNormalFunction, @@ -273,6 +307,34 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( return result; } +void UnlinkedFunctionExecutable::decodeCodeBlockFromCacheRecord(VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind, WriteBarrier& slot) +{ + if (!m_isGeneratedFromCache || m_cachedRecordOffset <= 0) + return; + RefPtr provider = source.provider(); + if (!provider) + return; + RefPtr cachedBytecode = provider->cachedBytecode(); + if (!cachedBytecode || static_cast(m_cachedRecordOffset) >= cachedBytecode->size()) + return; + Ref decoder = vm.ensureBytecodeCacheDecoder(cachedBytecode.releaseNonNull(), WTF::move(provider)); + decodeFunctionCodeBlockFromExecutableRecord(decoder.get(), m_cachedRecordOffset, specializationKind, slot, *this); +} + +UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::tryRedecodeCodeBlock(VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind) +{ + if (m_isCached) + return nullptr; // still holds its decoder and offsets: the ordinary cached path applies + DeferGC deferGC(vm); + auto& slot = specializationKind == CodeSpecializationKind::CodeForCall ? m_unlinkedCodeBlockForCall : m_unlinkedCodeBlockForConstruct; + decodeCodeBlockFromCacheRecord(vm, source, specializationKind, slot); + if (!slot) + return nullptr; + vm.writeBarrier(this); + vm.heap.unlinkedFunctionExecutableSpaceAndSet.set.add(this); + return slot.get(); +} + void UnlinkedFunctionExecutable::decodeCachedCodeBlocks(VM& vm) { ASSERT(m_isCached); @@ -296,6 +358,8 @@ void UnlinkedFunctionExecutable::decodeCachedCodeBlocks(VM& vm) WTF::storeStoreFence(); m_isCached = false; vm.writeBarrier(this); + // Registered so deleteAllCode / jettisoning can clear these too; they re-decode from the cache on next use. + vm.heap.unlinkedFunctionExecutableSpaceAndSet.set.add(this); } UnlinkedFunctionExecutable::RareData& UnlinkedFunctionExecutable::ensureRareDataSlow() diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h index b59b2a846eaa..4fc21a177e63 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h @@ -138,6 +138,9 @@ class UnlinkedFunctionExecutable final : public JSCell { void recordParse(CodeFeatures features, LexicallyScopedFeatures lexicallyScopedFeatures, bool hasCapturedVariables) { + // A reparse yields the same answers; don't store (and dirty a snapshot page) unless something actually changed. + if (m_features == features && m_lexicallyScopedFeatures == lexicallyScopedFeatures && m_hasCapturedVariables == static_cast(hasCapturedVariables)) + return; m_features = features; m_lexicallyScopedFeatures = lexicallyScopedFeatures; m_hasCapturedVariables = hasCapturedVariables; @@ -281,12 +284,17 @@ class UnlinkedFunctionExecutable final : public JSCell { DECLARE_VISIT_CHILDREN; void decodeCachedCodeBlocks(VM&); + UnlinkedFunctionCodeBlock* tryRedecodeCodeBlock(VM&, const SourceCode&, CodeSpecializationKind); + void decodeCodeBlockFromCacheRecord(VM&, const SourceCode&, CodeSpecializationKind, WriteBarrier&); // caller holds a DeferGC bool codeBlockEdgeMayBeWeak() const { - // Currently, bytecode cache assumes that the tree of UnlinkedFunctionExecutable and UnlinkedCodeBlock will not be destroyed while the parent is live. - // Bytecode cache uses this asumption to avoid duplicate materialization by bookkeeping the heap cells in the offste-to-pointer map. - return VM::useUnlinkedCodeBlockJettisoning() && !m_isGeneratedFromCache; + if (!VM::useUnlinkedCodeBlockJettisoning()) + return false; + if (!m_isGeneratedFromCache) + return true; + // While m_isCached, the code block slots hold the Decoder / cache offsets rather than code blocks. + return !m_isCached && Options::useUnlinkedCodeBlockJettisoningForBytecodeCache(); } unsigned m_firstLineOffset : 31; @@ -335,6 +343,8 @@ class UnlinkedFunctionExecutable final : public JSCell { Identifier m_name; Identifier m_ecmaName; + // Offset of this executable's CachedFunctionExecutable record in the provider's CachedBytecode (0 = none); lets a cleared code block be re-decoded instead of re-parsed. + int32_t m_cachedRecordOffset { 0 }; RareData& ensureRareData() { @@ -353,7 +363,7 @@ class UnlinkedFunctionExecutable final : public JSCell { }; #if !ASSERT_ENABLED -static_assert(sizeof(UnlinkedFunctionExecutable) <= 96, "UnlinkedFunctionExecutable needs to be small"); +static_assert(sizeof(UnlinkedFunctionExecutable) <= 104, "UnlinkedFunctionExecutable needs to be small"); #endif } // namespace JSC diff --git a/Source/JavaScriptCore/runtime/CachePayload.h b/Source/JavaScriptCore/runtime/CachePayload.h index 634350ac68e0..d2293947a87c 100644 --- a/Source/JavaScriptCore/runtime/CachePayload.h +++ b/Source/JavaScriptCore/runtime/CachePayload.h @@ -38,6 +38,9 @@ class CachePayload { JS_EXPORT_PRIVATE static CachePayload makeMallocPayload(MallocSpan&&); JS_EXPORT_PRIVATE static CachePayload makePayloadWithDestructor(std::span, Destructor&&); JS_EXPORT_PRIVATE static CachePayload makeEmptyPayload(); + // Only an explicit embedder promise makes a payload borrowable; nothing is inferred from how it is stored. + bool isPersistent() const { return m_isPersistent; } + void setIsPersistent() { m_isPersistent = true; } // embedder promises the bytes outlive every VM use (e.g. a section of the running executable) JS_EXPORT_PRIVATE CachePayload(CachePayload&&); JS_EXPORT_PRIVATE ~CachePayload(); @@ -50,6 +53,7 @@ class CachePayload { explicit CachePayload(DataType&&, Destructor&& = {nullptr}); DataType m_data; + bool m_isPersistent { false }; Destructor m_destructor { nullptr }; }; diff --git a/Source/JavaScriptCore/runtime/CachedBytecode.h b/Source/JavaScriptCore/runtime/CachedBytecode.h index 99669b1dfba5..e8564f77c67c 100644 --- a/Source/JavaScriptCore/runtime/CachedBytecode.h +++ b/Source/JavaScriptCore/runtime/CachedBytecode.h @@ -71,6 +71,8 @@ class CachedBytecode : public RefCounted { std::span span() const LIFETIME_BOUND { return m_payload.span(); } size_t size() const { return m_payload.size(); } + bool payloadIsPersistent() const { return m_payload.isPersistent(); } + void setPayloadIsPersistent() { m_payload.setIsPersistent(); } bool hasUpdates() const { return !m_updates.isEmpty(); } size_t sizeForUpdate() const { return m_size; } diff --git a/Source/JavaScriptCore/runtime/CachedTypes.cpp b/Source/JavaScriptCore/runtime/CachedTypes.cpp index 261fc8b0e5e8..3b15f31b778d 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.cpp +++ b/Source/JavaScriptCore/runtime/CachedTypes.cpp @@ -25,6 +25,7 @@ #include "config.h" #include "CachedTypes.h" +#include #include "BaselineJITCode.h" #include "BuiltinNames.h" @@ -56,6 +57,11 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { +bool Decoder::canBorrowPayload() const +{ + return Options::useBorrowedBytecodeFromCache() && m_cachedBytecode->payloadIsPersistent(); +} + namespace Yarr { enum class Flags : uint16_t; } @@ -293,8 +299,10 @@ Decoder::Decoder(VM& vm, Ref cachedBytecode, RefPtr Decoder::create(VM& vm, Ref cachedBytecode, RefPtr provider) @@ -349,13 +357,14 @@ void Decoder::setHandleForTDZEnvironment(CompactTDZEnvironment* environment, con void Decoder::addLeafExecutable(const UnlinkedFunctionExecutable* executable, ptrdiff_t offset) { + if (!m_recordsLeafExecutables) + return; m_cachedBytecode->leafExecutables().add(executable, offset); } -template -void Decoder::addFinalizer(const Functor& fn) +void Decoder::addFinalizer(void* ptr, void (*finalizer)(void*)) { - m_finalizers.append(fn); + m_finalizers.append({ ptr, finalizer }); } RefPtr Decoder::provider() const @@ -487,6 +496,11 @@ class CachedArray : public VariableLengthObject { for (unsigned i = 0; i < size; ++i) ::JSC::decode(decoder, buffer[i], array[i], args...); } + const T* borrow() const + { + static_assert(std::is_trivially_copyable_v, "borrowing is a raw view of the encoded bytes"); + return this->isEmpty() ? nullptr : this->template buffer(); + } }; template> @@ -512,8 +526,11 @@ class CachedPtr : public VariableLengthObject { encoder.cachePtr(src, encoder.offsetOf(cachedObject)); } + // Only for objects whose lifetime the Decoder itself guarantees (CachedRefPtr adds a finalizer holding a ref; + // TDZ environments are owned by the VM map or a finalizer). Uniquely-owned objects and GC cells can die while + // the Decoder lives, so they must always be decoded fresh via decode(). template - Source* decode(Decoder& decoder, bool& isNewAllocation, Args&&... args) const + Source* decodeShared(Decoder& decoder, bool& isNewAllocation, Args&&... args) const { if (this->isEmpty()) { isNewAllocation = false; @@ -535,8 +552,9 @@ class CachedPtr : public VariableLengthObject { template Source* decode(Decoder& decoder, Args&&... args) const { - bool unusedIsNewAllocation; - return decode(decoder, unusedIsNewAllocation, std::forward(args)...); + if (this->isEmpty()) + return nullptr; + return get()->decode(decoder, std::forward(args)...); } const T* NODELETE operator->() const { return get(); } @@ -570,12 +588,12 @@ class CachedRefPtr : public CachedObject> { RefPtr decode(Decoder& decoder) const { bool isNewAllocation; - Source* decodedPtr = m_ptr.decode(decoder, isNewAllocation); + Source* decodedPtr = m_ptr.decodeShared(decoder, isNewAllocation); if (!decodedPtr) return nullptr; if (isNewAllocation) { - decoder.addFinalizer([=] { - WTF::DefaultRefDerefTraits::derefIfNotNull(decodedPtr); + decoder.addFinalizer(decodedPtr, [](void* ptr) { + WTF::DefaultRefDerefTraits::derefIfNotNull(static_cast(ptr)); }); } auto result = adoptRef(decodedPtr); @@ -634,6 +652,14 @@ class CachedVector : public VariableLengthObject, InlineCap ::JSC::encode(encoder, buffer[i], vector[i]); } + std::span borrow() const // raw view of the encoded elements (only meaningful for trivially-encoded T like uint8_t) + { + static_assert(std::is_trivially_copyable_v, "borrowing is a raw view of the encoded bytes"); + if (!m_size) + return { }; + return { this->template buffer(), m_size }; + } + template void decode(Decoder& decoder, VectorContainer& vector, Args... args) const { @@ -1066,6 +1092,8 @@ class CachedExpressionInfo : public CachedObject { std::unique_ptr decode(Decoder& decoder) const { + if (decoder.canBorrowPayload() && !m_storage.isEmpty()) + return ExpressionInfo::createBorrowed(m_numberOfChapters, m_numberOfEncodedInfo, m_numberOfEncodedInfoExtensions, m_storage.borrow()); auto info = ExpressionInfo::createUninitialized(m_numberOfChapters, m_numberOfEncodedInfo, m_numberOfEncodedInfoExtensions); m_storage.decode(decoder, info->payload(), info->payloadSize()); return info; @@ -1172,7 +1200,7 @@ class CachedCompactTDZEnvironmentMapHandle : public CachedObjectget(environment, isNewEntry); if (!isNewEntry) { - decoder.addFinalizer([=] { - delete environment; + decoder.addFinalizer(environment, [](void* ptr) { + delete static_cast(ptr); }); } decoder.setHandleForTDZEnvironment(environment, handle); @@ -1514,11 +1542,16 @@ class CachedInstructionStream : public CachedObject { public: void encode(Encoder& encoder, const JSInstructionStream& stream) { + RELEASE_ASSERT(!stream.isBorrowed()); // a borrowed stream's bytes live in the payload being read, not in m_instructions m_instructions.encode(encoder, stream.m_instructions); } JSInstructionStream* decode(Decoder& decoder) const { + if (decoder.canBorrowPayload()) { + // The cache outlives the VM's use of it (mmap'd / embedded): point at the bytes instead of copying them. + return new JSInstructionStream(m_instructions.borrow(), JSInstructionStream::Borrow); + } Vector instructionsVector; m_instructions.decode(decoder, instructionsVector); return new JSInstructionStream(WTF::move(instructionsVector)); @@ -2484,9 +2517,10 @@ ALWAYS_INLINE UnlinkedFunctionExecutable::UnlinkedFunctionExecutable(Decoder& de if (!cachedExecutable.unlinkedCodeBlockForCall().isEmpty() || !cachedExecutable.unlinkedCodeBlockForConstruct().isEmpty()) { checkBounds(m_cachedCodeBlockForCallOffset, cachedExecutable.unlinkedCodeBlockForCall()); checkBounds(m_cachedCodeBlockForConstructOffset, cachedExecutable.unlinkedCodeBlockForConstruct()); - if (m_isCached) + if (m_isCached) { m_decoder = &decoder; - else + m_cachedRecordOffset = static_cast(decoder.offsetOf(&cachedExecutable)); + } else m_decoder = nullptr; } @@ -2761,7 +2795,7 @@ UnlinkedCodeBlock* decodeCodeBlockImpl(VM& vm, const SourceCodeKey& key, Ref(cachedBytecode->span().data()); - Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode), &key.source().provider()); + Ref decoder = vm.ensureBytecodeCacheDecoder(WTF::move(cachedBytecode), &key.source().provider()); std::pair entry; { DeferGC deferGC(vm); @@ -2796,6 +2830,30 @@ void decodeFunctionCodeBlock(Decoder& decoder, int32_t cachedFunctionCodeBlockOf cachedCodeBlock->decode(decoder, codeBlock, owner); } +void decodeFunctionCodeBlockForReDecode(Decoder& decoder, int32_t cachedFunctionCodeBlockOffset, WriteBarrier& codeBlock, const JSCell* owner) +{ + SetForScope reDecoding(decoder.m_recordsLeafExecutables, false); + decodeFunctionCodeBlock(decoder, cachedFunctionCodeBlockOffset, codeBlock, owner); +} + +void decodeFunctionCodeBlockFromExecutableRecord(Decoder& decoder, int32_t cachedFunctionExecutableOffset, CodeSpecializationKind kind, WriteBarrier& codeBlock, const UnlinkedFunctionExecutable& executable) +{ + ASSERT(decoder.vm().heap.isDeferred()); + if (cachedFunctionExecutableOffset <= 0 || static_cast(cachedFunctionExecutableOffset) + sizeof(CachedFunctionExecutable) > decoder.size()) + return; + auto* cachedExecutable = static_cast(decoder.ptrForOffsetFromBase(cachedFunctionExecutableOffset)); + // The offset was recorded against the payload this executable was first decoded from; the provider's payload could have been + // replaced or rewritten since, so the record must describe this very function before anything in it is trusted. + if (cachedExecutable->startOffset() != executable.startOffset() || cachedExecutable->unlinkedFunctionStart() != executable.unlinkedFunctionStart() + || cachedExecutable->lineCount() != executable.lineCount() || cachedExecutable->parameterCount() != executable.parameterCount()) + return; + const auto& cachedCodeBlock = kind == CodeSpecializationKind::CodeForCall ? cachedExecutable->unlinkedCodeBlockForCall() : cachedExecutable->unlinkedCodeBlockForConstruct(); + if (cachedCodeBlock.isEmpty()) + return; + SetForScope reDecoding(decoder.m_recordsLeafExecutables, false); + cachedCodeBlock.decode(decoder, codeBlock, &executable); +} + } // namespace JSC WTF_ALLOW_UNSAFE_BUFFER_USAGE_END diff --git a/Source/JavaScriptCore/runtime/CachedTypes.h b/Source/JavaScriptCore/runtime/CachedTypes.h index 3ce6592e5b92..6465292056e1 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.h +++ b/Source/JavaScriptCore/runtime/CachedTypes.h @@ -82,6 +82,7 @@ class Decoder : public RefCounted { public: static Ref create(VM&, Ref, RefPtr = nullptr); + bool canBorrowPayload() const; // the embedder promised the payload outlives every use, so decoded objects may alias it ~Decoder(); @@ -96,19 +97,24 @@ class Decoder : public RefCounted { void setHandleForTDZEnvironment(CompactTDZEnvironment*, const CompactTDZEnvironmentMap::Handle&); void addLeafExecutable(const UnlinkedFunctionExecutable*, ptrdiff_t); RefPtr NODELETE provider() const; + CachedBytecode& cachedBytecode() const { return m_cachedBytecode.get(); } - template - void addFinalizer(const Functor&); + void addFinalizer(void*, void (*)(void*)); + void setIsRegisteredWithVM() { m_isRegisteredWithVM = true; } private: + friend void decodeFunctionCodeBlockForReDecode(Decoder&, int32_t, WriteBarrier&, const JSCell*); + friend void decodeFunctionCodeBlockFromExecutableRecord(Decoder&, int32_t, CodeSpecializationKind, WriteBarrier&, const UnlinkedFunctionExecutable&); Decoder(VM&, Ref, RefPtr); VM& m_vm; const Ref m_cachedBytecode; + bool m_recordsLeafExecutables { true }; // cleared for the duration of a re-decode: nothing decoded then is ever written back UncheckedKeyHashMap m_offsetToPtrMap; - Vector> m_finalizers; + Vector> m_finalizers; UncheckedKeyHashMap m_environmentToHandleMap; RefPtr m_provider; + bool m_isRegisteredWithVM { false }; }; JS_EXPORT_PRIVATE RefPtr encodeCodeBlock(VM&, const SourceCodeKey&, const UnlinkedCodeBlock*); @@ -127,6 +133,8 @@ std::optional decodeSourceCodeKey(VM& vm, Ref cac JS_EXPORT_PRIVATE RefPtr encodeFunctionCodeBlock(VM&, const UnlinkedFunctionCodeBlock*, BytecodeCacheError&); JS_EXPORT_PRIVATE void decodeFunctionCodeBlock(Decoder&, int32_t cachedFunctionCodeBlockOffset, WriteBarrier&, const JSCell*); +JS_EXPORT_PRIVATE void decodeFunctionCodeBlockForReDecode(Decoder&, int32_t cachedFunctionCodeBlockOffset, WriteBarrier&, const JSCell* owner); // as decodeFunctionCodeBlock, for code that is being decoded again: records nothing +JS_EXPORT_PRIVATE void decodeFunctionCodeBlockFromExecutableRecord(Decoder&, int32_t cachedFunctionExecutableOffset, CodeSpecializationKind, WriteBarrier&, const UnlinkedFunctionExecutable&); bool isCachedBytecodeStillValid(VM&, Ref, const SourceCodeKey&, SourceCodeType); From 287afcc0b3f6c855e928a92efb39db485505de8e Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 10 Aug 2026 06:05:18 -0700 Subject: [PATCH 3/4] VM::didRestoreFromStartupSnapshot(): what a process resumed from a startup snapshot has to recompute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stack bounds and the cached stack top belong to the process that took the snapshot; so do the threads behind AutomaticThreads (GC helpers, JIT worklists — they are restarted on the next notify), ParkingLot entries, the random number generator's state, ICU objects held by the Intl and date caches, and the regexp JIT code. Per-heap tables that take inserts every run are re-homed so their growth does not dirty snapshot pages, and allocation pacing starts over from an empty heap: the limits in the snapshot were computed while its cells were ordinary live cells, whereas now they are immortal and uncounted. Signal handling is kernel state: the Mach exception port and handler thread (or the sigaction table elsewhere) that WTF installed belong to the process that took the snapshot, while WTF's own record says they are in place, so a wasm out-of-bounds access in the resumed process was a crash rather than a trap. SignalHandlers::reinstallAfterSnapshotRestore() installs them again for the resumed process, after which the config is frozen again (it is copied in writable, since this process never froze it). On the way in, completeAllJITPlansBeforeStartupSnapshot() also cancels and joins wasm compilation plans: BBQ/OMG tier-ups still running after a warm-up were installing code into the executable pool while the snapshot was being written, so the snapshot held the module's pointers to code it did not contain. The OS random source is reopened before the generator is stirred: on Linux it holds a /dev/urandom descriptor, and the number restored with it belongs to the process that took the snapshot, so the stir read from whatever the resumed process happened to have there and could block forever. Likewise the thread suspend/resume signal handler (SIGPWR on Linux), installed once by initializePlatformThreading(): the resumed process was killed by the signal the first time something suspended a thread. StackBounds (Linux) caches the process's original environ pointer in a static to keep the environment vector out of the main thread's bounds; resumed from a snapshot, that is an address on the build process's stack, and whenever ASLR put the new stack over that range the origin was clamped below the live frames (sanitizeStackForVM's assertion, ~1% of launches). Adopting the thread forgets the cached value so it is re-read from this process's environ. --- .../JavaScriptCore/runtime/IntlCollator.cpp | 46 ++++++- Source/JavaScriptCore/runtime/IntlCollator.h | 8 +- .../runtime/IntlDateTimeFormat.cpp | 35 +++++ .../runtime/IntlDateTimeFormat.h | 10 +- .../runtime/IntlDisplayNames.cpp | 38 ++++++ .../JavaScriptCore/runtime/IntlDisplayNames.h | 7 + .../runtime/IntlDurationFormat.cpp | 59 ++++++--- .../runtime/IntlDurationFormat.h | 6 + .../JavaScriptCore/runtime/IntlListFormat.cpp | 39 ++++++ .../JavaScriptCore/runtime/IntlListFormat.h | 8 ++ .../runtime/IntlNumberFormat.cpp | 35 +++++ .../JavaScriptCore/runtime/IntlNumberFormat.h | 5 + .../runtime/IntlPluralRules.cpp | 43 ++++++ .../JavaScriptCore/runtime/IntlPluralRules.h | 8 ++ .../runtime/IntlRelativeTimeFormat.cpp | 51 +++++++- .../runtime/IntlRelativeTimeFormat.h | 5 + .../runtime/IntlSegmentIterator.cpp | 32 ++++- .../runtime/IntlSegmentIterator.h | 10 +- .../JavaScriptCore/runtime/IntlSegmenter.cpp | 25 +++- Source/JavaScriptCore/runtime/IntlSegmenter.h | 9 +- .../JavaScriptCore/runtime/IntlSegments.cpp | 32 ++++- Source/JavaScriptCore/runtime/IntlSegments.h | 9 +- Source/JavaScriptCore/runtime/JSDateMath.cpp | 9 ++ Source/JavaScriptCore/runtime/JSDateMath.h | 5 + Source/JavaScriptCore/runtime/VM.cpp | 83 ++++++++++++ Source/JavaScriptCore/runtime/VM.h | 20 +++ Source/WTF/wtf/AutomaticThread.cpp | 43 ++++++ Source/WTF/wtf/AutomaticThread.h | 3 + .../WTF/wtf/CryptographicallyRandomNumber.cpp | 11 ++ .../WTF/wtf/CryptographicallyRandomNumber.h | 7 + Source/WTF/wtf/OSRandomSource.cpp | 19 ++- Source/WTF/wtf/OSRandomSource.h | 3 + Source/WTF/wtf/ParkingLot.cpp | 13 ++ Source/WTF/wtf/ParkingLot.h | 6 + Source/WTF/wtf/RandomDevice.cpp | 10 ++ Source/WTF/wtf/RandomDevice.h | 3 + Source/WTF/wtf/StackBounds.cpp | 20 ++- Source/WTF/wtf/StackBounds.h | 5 + Source/WTF/wtf/Threading.h | 12 ++ Source/WTF/wtf/posix/ThreadingPOSIX.cpp | 88 +++++++++---- Source/WTF/wtf/threads/Signals.cpp | 122 +++++++++++++----- Source/WTF/wtf/threads/Signals.h | 3 + 42 files changed, 904 insertions(+), 101 deletions(-) diff --git a/Source/JavaScriptCore/runtime/IntlCollator.cpp b/Source/JavaScriptCore/runtime/IntlCollator.cpp index 6d745aa0e534..405f8d6184e9 100644 --- a/Source/JavaScriptCore/runtime/IntlCollator.cpp +++ b/Source/JavaScriptCore/runtime/IntlCollator.cpp @@ -28,6 +28,7 @@ #include "config.h" #include "IntlCollator.h" +#include "TopExceptionScope.h" #include "IntlObjectInlines.h" #include "JSBoundFunction.h" #include "JSCInlines.h" @@ -224,8 +225,21 @@ void IntlCollator::initializeCollator(JSGlobalObject* globalObject, JSValue loca } dataLogLnIf(IntlCollatorInternal::verbose, "locale:(", resolved.locale, "),dataLocaleWithExtensions:(", dataLocaleWithExtensions, ")"); + m_icuLocale = dataLocaleWithExtensions; +#if USE(BUN_JSC_ADDITIONS) + m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); +#endif + scope.release(); // whatever the opener throws is this initialization's to propagate + openCollator(globalObject, ignorePunctuation); +} + +// Opens and configures the ICU collator from the resolved fields. Called once when the object is initialized and again in a +// process restored from a snapshot, whose ICU is not the one the existing handle came from. +void IntlCollator::openCollator(JSGlobalObject* globalObject, TriState ignorePunctuation) +{ + auto scope = DECLARE_THROW_SCOPE(globalObject->vm()); UErrorCode status = U_ZERO_ERROR; - m_collator = std::unique_ptr(ucol_open(dataLocaleWithExtensions.data(), &status)); + m_collator = std::unique_ptr(ucol_open(m_icuLocale.data(), &status)); if (U_FAILURE(status)) { throwTypeError(globalObject, scope, "failed to initialize Collator"_s); return; @@ -281,10 +295,34 @@ void IntlCollator::initializeCollator(JSGlobalObject* globalObject, JSValue loca } } +UCollator* IntlCollator::collatorForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_startupSnapshotEpoch != vm.startupSnapshotEpoch()) [[unlikely]] { + (void)m_collator.release(); // the handle belongs to the process that built the snapshot; its ICU is gone + m_canDoASCIIUCADUCETComparison = TriState::Indeterminate; // a verdict about the old ICU's rules; this machine's may differ + { + auto catchScope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + const_cast(this)->openCollator(globalObject, m_ignorePunctuation ? TriState::True : TriState::False); + if (catchScope.exception()) [[unlikely]] { + (void)catchScope.tryClearException(); + RELEASE_ASSERT_WITH_MESSAGE(false, "Intl.Collator could not reopen its ICU collator in the restored process (different ICU?)"); + } + } + m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); + } +#else + UNUSED_PARAM(globalObject); +#endif + return m_collator.get(); +} + // https://tc39.es/ecma402/#sec-collator-comparestrings UCollationResult IntlCollator::compareStrings(JSGlobalObject* globalObject, StringView x, StringView y) const { - ASSERT(m_collator); + UCollator* collator = collatorForThisProcess(globalObject); // before the ASCII probe below, which uses the handle + ASSERT(collator); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -304,14 +342,14 @@ UCollationResult IntlCollator::compareStrings(JSGlobalObject* globalObject, Stri if (x.is8Bit() && y.is8Bit() && x.containsOnlyASCII() && y.containsOnlyASCII()) { auto xCharacters = byteCast(x.span8()); auto yCharacters = byteCast(y.span8()); - return ucol_strcollUTF8(m_collator.get(), xCharacters.data(), xCharacters.size(), yCharacters.data(), yCharacters.size(), &status); + return ucol_strcollUTF8(collator, xCharacters.data(), xCharacters.size(), yCharacters.data(), yCharacters.size(), &status); } return std::nullopt; }()); if (!result) - result = ucol_strcoll(m_collator.get(), x.upconvertedCharacters(), x.length(), y.upconvertedCharacters(), y.length()); + result = ucol_strcoll(collator, x.upconvertedCharacters(), x.length(), y.upconvertedCharacters(), y.length()); if (U_FAILURE(status)) { throwException(globalObject, scope, createError(globalObject, "Failed to compare strings."_s)); diff --git a/Source/JavaScriptCore/runtime/IntlCollator.h b/Source/JavaScriptCore/runtime/IntlCollator.h index a6f93ce15d99..519882b72b77 100644 --- a/Source/JavaScriptCore/runtime/IntlCollator.h +++ b/Source/JavaScriptCore/runtime/IntlCollator.h @@ -61,6 +61,8 @@ class IntlCollator final : public JSNonFinalObject { DECLARE_VISIT_CHILDREN; void initializeCollator(JSGlobalObject*, JSValue locales, JSValue optionsValue); + void openCollator(JSGlobalObject*, TriState ignorePunctuation); + UCollator* collatorForThisProcess(JSGlobalObject*) const; UCollationResult compareStrings(JSGlobalObject*, StringView, StringView) const; JSObject* resolvedOptions(JSGlobalObject*) const; @@ -100,7 +102,11 @@ class IntlCollator final : public JSNonFinalObject { static ASCIILiteral caseFirstString(CaseFirst); WriteBarrier m_boundCompare; - std::unique_ptr m_collator; + mutable std::unique_ptr m_collator; + CString m_icuLocale; +#if USE(BUN_JSC_ADDITIONS) + mutable unsigned m_startupSnapshotEpoch { 0 }; +#endif String m_locale; String m_collation; diff --git a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp index 48fe49442564..d05d32573190 100644 --- a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp +++ b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp @@ -1088,6 +1088,11 @@ void IntlDateTimeFormat::initializeDateTimeFormat(JSGlobalObject* globalObject, UErrorCode status = U_ZERO_ERROR; String timeZoneForICU = impl->m_timeZone.toICUString(); impl->m_dateFormat = openDateFormat(dataLocaleWithExtensions, timeZoneForICU, patternBuffer.span(), status); +#if USE(BUN_JSC_ADDITIONS) + impl->m_icuPattern = patternBuffer; + impl->m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); + m_intervalFormatEpoch = impl->m_startupSnapshotEpoch; +#endif if (U_FAILURE(status)) [[unlikely]] { throwTypeError(globalObject, scope, "failed to initialize DateTimeFormat"_s); return; @@ -1394,8 +1399,34 @@ static void NODELETE replaceNarrowNoBreakSpaceOrThinSpaceWithNormalSpace(Contain } // https://tc39.es/proposal-temporal/#sec-formatdatetime + +void IntlDateTimeFormat::ensureICUObjectsForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_intervalFormatEpoch != vm.startupSnapshotEpoch()) [[unlikely]] { + (void)m_dateIntervalFormat.release(); // rebuilt on demand by formatRange + m_intervalFormatEpoch = vm.startupSnapshotEpoch(); + } + auto* impl = const_cast(m_impl.get()); + if (!impl || impl->m_startupSnapshotEpoch == vm.startupSnapshotEpoch()) [[likely]] + return; + // Every handle here was opened by the process that built the snapshot, against an ICU this process does not share (on macOS + // it is the system's). The main formatter is opened again from what it was opened with; the ones built lazily are dropped + // and come back on demand. The old handles are leaked on purpose: closing them would hand foreign memory to this ICU. + (void)impl->m_dateFormat.release(); + (void)impl->m_temporalFormatterCache.release(); // its formatters come back on demand + UErrorCode status = U_ZERO_ERROR; + impl->m_dateFormat = openDateFormat(impl->m_dataLocaleWithExtensions, impl->m_timeZone.toICUString(), impl->m_icuPattern.span(), status); + if (U_SUCCESS(status)) + impl->m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); +#else + UNUSED_PARAM(globalObject); +#endif +} JSValue IntlDateTimeFormat::format(JSGlobalObject* globalObject, double value) const { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); @@ -1988,6 +2019,7 @@ static JSValue buildFormattedDateIntervalParts(JSGlobalObject* globalObject, con // https://tc39.es/proposal-temporal/#sec-formatdatetimerangetoparts JSValue IntlDateTimeFormat::formatRangeToParts(JSGlobalObject* globalObject, JSValue xValue, JSValue yValue) { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -2384,6 +2416,7 @@ std::unique_ptr IntlDateTim // FormatDateTime(dateTimeFormat, x) — dispatches through HandleDateTimeValue for Temporal objects. JSValue IntlDateTimeFormat::format(JSGlobalObject* globalObject, JSValue x) const { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -2414,6 +2447,7 @@ JSValue IntlDateTimeFormat::format(JSGlobalObject* globalObject, JSValue x) cons // https://tc39.es/proposal-temporal/#sec-formatdatetimetoparts JSValue IntlDateTimeFormat::formatToParts(JSGlobalObject* globalObject, JSValue x) const { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -2467,6 +2501,7 @@ IntlDateTimeFormat::createTemporalIntervalFormat(UDateFormat* tempFormat, Tempor // https://tc39.es/proposal-temporal/#sec-formatdatetimerange JSValue IntlDateTimeFormat::formatRange(JSGlobalObject* globalObject, JSValue xValue, JSValue yValue) { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h index 6805a22d0ff6..6674a178ddde 100644 --- a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h +++ b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h @@ -121,6 +121,7 @@ class IntlDateTimeFormat final : public JSNonFinalObject { JSValue formatRange(JSGlobalObject*, JSValue startDate, JSValue endDate); JSValue formatRangeToParts(JSGlobalObject*, JSValue startDate, JSValue endDate); JSObject* resolvedOptions(JSGlobalObject*) const; + void ensureICUObjectsForThisProcess(JSGlobalObject*) const; // snapshot restore: the ICU handles came from another process static bool isTemporalObject(JSValue); static bool sameTemporalType(JSValue, JSValue); @@ -213,7 +214,10 @@ class IntlDateTimeFormat final : public JSNonFinalObject { static String buildSkeleton(Weekday, Era, Year, Month, Day, TriState, HourCycle, Hour, DayPeriod, Minute, Second, unsigned, TimeZoneName); WriteBarrier m_boundFormat; - std::unique_ptr m_dateIntervalFormat; + mutable std::unique_ptr m_dateIntervalFormat; +#if USE(BUN_JSC_ADDITIONS) + mutable unsigned m_intervalFormatEpoch { 0 }; // this object's own lazily-built handle; the shared impl has its own epoch +#endif RefPtr m_impl; }; @@ -258,6 +262,10 @@ class IntlDateTimeFormatImpl : public RefCounted { IntlDateTimeFormat::DateTimeStyle m_dateStyle { IntlDateTimeFormat::DateTimeStyle::None }; IntlDateTimeFormat::DateTimeStyle m_timeStyle { IntlDateTimeFormat::DateTimeStyle::None }; bool m_anyPresent { false }; +#if USE(BUN_JSC_ADDITIONS) + Vector m_icuPattern; // the pattern m_dateFormat was opened with, so it can be opened again after a snapshot restore + unsigned m_startupSnapshotEpoch { 0 }; +#endif Vector m_userSkeleton; // user's explicit options as skeleton, before defaults injection; used by computeGetDateTimeFormat std::unique_ptr m_dateFormat; mutable std::unique_ptr m_temporalFormatterCache; diff --git a/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp b/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp index ce3def9f190e..ff23773ff351 100644 --- a/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp +++ b/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp @@ -26,6 +26,7 @@ #include "config.h" #include "IntlDisplayNames.h" +#include "TopExceptionScope.h" #include "IntlCache.h" #include "IntlObjectInlines.h" #include "JSCInlines.h" @@ -105,6 +106,18 @@ void IntlDisplayNames::initializeDisplayNames(JSGlobalObject* globalObject, JSVa m_languageDisplay = intlOption(globalObject, options, vm.propertyNames->languageDisplay, { { "dialect"_s, LanguageDisplay::Dialect }, { "standard"_s, LanguageDisplay::Standard } }, "languageDisplay must be either \"dialect\" or \"standard\""_s, LanguageDisplay::Dialect); RETURN_IF_EXCEPTION(scope, void()); +#if USE(BUN_JSC_ADDITIONS) + m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); +#endif + scope.release(); // whatever the opener throws is this initialization's to propagate + openICUObjects(globalObject); +} + +// Builds the ICU objects from the resolved fields: once at initialization, and again in a process restored from a +// snapshot, whose ICU is not the one the existing handles came from. +void IntlDisplayNames::openICUObjects(JSGlobalObject* globalObject) +{ + auto scope = DECLARE_THROW_SCOPE(globalObject->vm()); UErrorCode status = U_ZERO_ERROR; UDisplayContext contexts[] = { @@ -134,9 +147,34 @@ void IntlDisplayNames::initializeDisplayNames(JSGlobalObject* globalObject, JSVa } } +void IntlDisplayNames::ensureICUObjectsForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_startupSnapshotEpoch == vm.startupSnapshotEpoch()) [[likely]] + return; + auto* self = const_cast(this); + // Opened by the process that built the snapshot; released, never closed, since this ICU never allocated them. + (void)self->m_displayNames.release(); + { + auto catchScope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + self->openICUObjects(globalObject); + if (catchScope.exception()) [[unlikely]] { + (void)catchScope.tryClearException(); + RELEASE_ASSERT_WITH_MESSAGE(false, "%s could not reopen its ICU objects in the restored process (different ICU?)", "IntlDisplayNames"); + } + } + if (self->m_displayNames) + self->m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); +#else + UNUSED_PARAM(globalObject); +#endif +} + // https://tc39.es/proposal-intl-displaynames/#sec-Intl.DisplayNames.prototype.of JSValue IntlDisplayNames::of(JSGlobalObject* globalObject, JSValue codeValue) const { + ensureICUObjectsForThisProcess(globalObject); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/runtime/IntlDisplayNames.h b/Source/JavaScriptCore/runtime/IntlDisplayNames.h index 17f257cb04a5..0c23a47a4514 100644 --- a/Source/JavaScriptCore/runtime/IntlDisplayNames.h +++ b/Source/JavaScriptCore/runtime/IntlDisplayNames.h @@ -55,6 +55,10 @@ class IntlDisplayNames final : public JSNonFinalObject { DECLARE_INFO; + + void openICUObjects(JSGlobalObject*); + + void ensureICUObjectsForThisProcess(JSGlobalObject*) const; void initializeDisplayNames(JSGlobalObject*, JSValue localesValue, JSValue optionsValue); JSValue of(JSGlobalObject*, JSValue) const; @@ -76,6 +80,9 @@ class IntlDisplayNames final : public JSNonFinalObject { using ULocaleDisplayNamesDeleter = ICUDeleter; std::unique_ptr m_displayNames; +#if USE(BUN_JSC_ADDITIONS) + unsigned m_startupSnapshotEpoch { 0 }; +#endif String m_locale; // FIXME: We should store it only when m_type is Currency. // https://bugs.webkit.org/show_bug.cgi?id=213773 diff --git a/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp b/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp index 8de3d534b903..018e2d4f08ce 100644 --- a/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp +++ b/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp @@ -194,6 +194,24 @@ static PropertyName NODELETE displayName(VM& vm, TemporalUnit unit) } // https://tc39.es/proposal-intl-duration-format/#sec-Intl.DurationFormat +static UListFormatterWidth toUListFormatterWidth(IntlDurationFormat::Style style) +{ + // 6. Let listStyle be durationFormat.[[Style]]. + // 7. If listStyle is "digital", then + // a. Set listStyle to "short". + // 8. Perform ! CreateDataPropertyOrThrow(lfOpts, "style", listStyle). + switch (style) { + case IntlDurationFormat::Style::Long: + return ULISTFMT_WIDTH_WIDE; + case IntlDurationFormat::Style::Short: + case IntlDurationFormat::Style::Digital: + return ULISTFMT_WIDTH_SHORT; + case IntlDurationFormat::Style::Narrow: + return ULISTFMT_WIDTH_NARROW; + } + return ULISTFMT_WIDTH_WIDE; +} + void IntlDurationFormat::initializeDurationFormat(JSGlobalObject* globalObject, JSValue locales, JSValue optionsValue) { VM& vm = globalObject->vm(); @@ -264,22 +282,6 @@ void IntlDurationFormat::initializeDurationFormat(JSGlobalObject* globalObject, RETURN_IF_EXCEPTION(scope, void()); { - auto toUListFormatterWidth = [](Style style) { - // 6. Let listStyle be durationFormat.[[Style]]. - // 7. If listStyle is "digital", then - // a. Set listStyle to "short". - // 8. Perform ! CreateDataPropertyOrThrow(lfOpts, "style", listStyle). - switch (style) { - case Style::Long: - return ULISTFMT_WIDTH_WIDE; - case Style::Short: - case Style::Digital: - return ULISTFMT_WIDTH_SHORT; - case Style::Narrow: - return ULISTFMT_WIDTH_NARROW; - } - return ULISTFMT_WIDTH_WIDE; - }; // 5. Perform ! CreateDataPropertyOrThrow(lfOpts, "type", "unit"). UErrorCode status = U_ZERO_ERROR; @@ -289,6 +291,29 @@ void IntlDurationFormat::initializeDurationFormat(JSGlobalObject* globalObject, return; } } +#if USE(BUN_JSC_ADDITIONS) + m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); +#endif +} + +void IntlDurationFormat::ensureICUObjectsForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_startupSnapshotEpoch == vm.startupSnapshotEpoch()) [[likely]] + return; + auto* self = const_cast(this); + // Opened by the process that built the snapshot, against an ICU this process does not share: released, never closed. The + // per-unit number formatters are built lazily, so dropping the cache is enough for them; the list formatter is opened again. + (void)self->m_formatterCache.release(); + (void)self->m_listFormat.release(); + UErrorCode status = U_ZERO_ERROR; + self->m_listFormat = std::unique_ptr(ulistfmt_openForType(m_locale.utf8().data(), ULISTFMT_TYPE_UNITS, toUListFormatterWidth(m_style), &status)); + if (U_SUCCESS(status)) + self->m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); +#else + UNUSED_PARAM(globalObject); +#endif } const String& IntlDurationFormat::numberingSystem() const @@ -690,6 +715,7 @@ UNumberFormatter* IntlDurationFormat::createNumberFormatterIfNecessary(JSGlobalO // https://tc39.es/proposal-intl-duration-format/#sec-Intl.DurationFormat.prototype.format JSValue IntlDurationFormat::format(JSGlobalObject* globalObject, ISO8601::Duration duration) const { + ensureICUObjectsForThisProcess(globalObject); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -733,6 +759,7 @@ JSValue IntlDurationFormat::format(JSGlobalObject* globalObject, ISO8601::Durati // https://tc39.es/proposal-intl-duration-format/#sec-Intl.DurationFormat.prototype.formatToParts JSValue IntlDurationFormat::formatToParts(JSGlobalObject* globalObject, ISO8601::Duration duration) const { + ensureICUObjectsForThisProcess(globalObject); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/runtime/IntlDurationFormat.h b/Source/JavaScriptCore/runtime/IntlDurationFormat.h index 45b656c8cad1..ee297d51f545 100644 --- a/Source/JavaScriptCore/runtime/IntlDurationFormat.h +++ b/Source/JavaScriptCore/runtime/IntlDurationFormat.h @@ -59,6 +59,8 @@ class IntlDurationFormat final : public JSNonFinalObject { DECLARE_VISIT_CHILDREN; + + void ensureICUObjectsForThisProcess(JSGlobalObject*) const; void initializeDurationFormat(JSGlobalObject*, JSValue localesValue, JSValue optionsValue); JSValue format(JSGlobalObject*, ISO8601::Duration) const; @@ -112,6 +114,10 @@ class IntlDurationFormat final : public JSNonFinalObject { }; std::unique_ptr m_listFormat; +#if USE(BUN_JSC_ADDITIONS) + + unsigned m_startupSnapshotEpoch { 0 }; +#endif mutable std::unique_ptr m_formatterCache; String m_locale; String m_dataLocale; diff --git a/Source/JavaScriptCore/runtime/IntlListFormat.cpp b/Source/JavaScriptCore/runtime/IntlListFormat.cpp index 4f8221b84cfa..65a456536c9b 100644 --- a/Source/JavaScriptCore/runtime/IntlListFormat.cpp +++ b/Source/JavaScriptCore/runtime/IntlListFormat.cpp @@ -27,6 +27,7 @@ #include "config.h" #include "IntlListFormat.h" +#include "TopExceptionScope.h" #include "IntlObjectInlines.h" #include "IntlPartObject.h" #include "IteratorOperations.h" @@ -109,6 +110,18 @@ void IntlListFormat::initializeListFormat(JSGlobalObject* globalObject, JSValue m_style = intlOption