diff --git a/CLAUDE.md b/CLAUDE.md index d074a3c4ca4e..c92f9ed9d416 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,6 +149,10 @@ High-performance memory allocator with: - Stack trace improvements - Better error reporting for development +5. **Startup Snapshots** + - `heap/Heap.cpp` (`freezeCurrentHeapAsImmortalStartupSnapshot`, `resetPacingAfterSnapshotRestore`), `runtime/VM.cpp` (`didRestoreFromStartupSnapshot`) + - Freezes the live heap as immortal so an embedder can write the process's memory out and resume later launches from it + ### USE_BUN_EVENT_LOOP Custom event loop implementation for Bun's runtime requirements diff --git a/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp b/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp index 932fbc882153..317f322a3e75 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp +++ b/Source/JavaScriptCore/bytecode/BytecodeRewriter.cpp @@ -49,6 +49,7 @@ void BytecodeRewriter::applyModification() m_writer.m_instructions.insertVector(insertion.index.bytecodeOffset, insertion.instructions.m_instructions); } } + m_writer.didMutateBuffer(); m_insertions.clear(); } diff --git a/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp b/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp index 8189d7c39efe..1819129a231b 100644 --- a/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp +++ b/Source/JavaScriptCore/bytecode/ExpressionInfo.cpp @@ -898,6 +898,14 @@ std::unique_ptr ExpressionInfo::createUninitialized(unsigned num return std::unique_ptr(new (allocation) ExpressionInfo(numberOfChapters, numberOfEncodedInfo, numberOfEncodedInfoExtensions)); } +std::unique_ptr ExpressionInfo::createBorrowed(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions, const unsigned* payload) +{ + void* allocation = FastMalloc::malloc(sizeof(ExpressionInfo)); + auto info = std::unique_ptr(new (allocation) ExpressionInfo(numberOfChapters, numberOfEncodedInfo, numberOfEncodedInfoExtensions)); + info->m_borrowedPayload = const_cast(payload); + return info; +} + ExpressionInfo::ExpressionInfo(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions) : m_numberOfChapters(numberOfChapters) , m_numberOfEncodedInfo(numberOfEncodedInfo) @@ -914,6 +922,8 @@ ExpressionInfo::ExpressionInfo(Vector&& chapters, Vector&& size_t ExpressionInfo::byteSize() const { + if (m_borrowedPayload) [[unlikely]] + return sizeof(ExpressionInfo); // the payload belongs to the cache mapping, not to this object return totalSizeInBytes(m_numberOfChapters, m_numberOfEncodedInfo, m_numberOfEncodedInfoExtensions); } diff --git a/Source/JavaScriptCore/bytecode/ExpressionInfo.h b/Source/JavaScriptCore/bytecode/ExpressionInfo.h index 8a79926a83de..d2935b87d8be 100644 --- a/Source/JavaScriptCore/bytecode/ExpressionInfo.h +++ b/Source/JavaScriptCore/bytecode/ExpressionInfo.h @@ -214,7 +214,7 @@ class ExpressionInfo { Chapter* chapters() const { - return std::bit_cast(this + 1); + return std::bit_cast(payload()); } EncodedInfo* encodedInfo() const @@ -239,10 +239,14 @@ class ExpressionInfo { unsigned* payload() const { + if (m_borrowedPayload) [[unlikely]] + return m_borrowedPayload; return std::bit_cast(this + 1); } static std::unique_ptr createUninitialized(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions); + // Header-only object whose (immutable) payload lives elsewhere, e.g. inside an mmap'd bytecode cache. + static std::unique_ptr createBorrowed(unsigned numberOfChapters, unsigned numberOfEncodedInfo, unsigned numberOfEncodedInfoExtensions, const unsigned* payload); static constexpr unsigned bitsPerWord = sizeof(unsigned) * CHAR_BIT; @@ -323,6 +327,7 @@ class ExpressionInfo { unsigned m_numberOfChapters; unsigned m_numberOfEncodedInfo; unsigned m_numberOfEncodedInfoExtensions; + unsigned* m_borrowedPayload { nullptr }; // Followed by the following which are allocated but are dynamically sized. // Chapter chapters[numberOfChapters]; // EncodedInfo encodedInfo[numberOfEncodedInfo + numberOfEncodedInfoExtensions]; diff --git a/Source/JavaScriptCore/bytecode/InstructionStream.h b/Source/JavaScriptCore/bytecode/InstructionStream.h index e094a85db136..f53b1ba85c1e 100644 --- a/Source/JavaScriptCore/bytecode/InstructionStream.h +++ b/Source/JavaScriptCore/bytecode/InstructionStream.h @@ -30,6 +30,7 @@ #include "Instruction.h" #include #include +#include WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -52,89 +53,90 @@ class InstructionStream { size_t sizeInBytes() const { - return m_instructions.size(); + return m_bytes.size(); } + size_t ownedSizeInBytes() const { return m_isBorrowed ? 0 : m_bytes.size(); } // what this object reports to the GC as its own using Offset = unsigned; private: - template + // Refs read through the stream's span (one indirection, exactly like reading through a reference to the buffer), so a + // ref minted while a writer is still appending keeps seeing the current bytes: the writer refreshes the span after + // every mutation of its buffer. + using Bytes = std::span; + +public: + class Ref; +private: class BaseRef { WTF_DEPRECATED_MAKE_FAST_ALLOCATED(BaseRef); template friend class InstructionStream; + template friend class InstructionStreamWriter; public: - BaseRef(const BaseRef& other) - : m_instructions(other.m_instructions) - , m_index(other.m_index) - { } - - void operator=(const BaseRef& other) - { - m_instructions = other.m_instructions; - m_index = other.m_index; - } - inline const InstructionType* operator->() const { return unwrap(); } inline const InstructionType* ptr() const { return unwrap(); } - bool operator==(const BaseRef& other) const + bool operator==(const BaseRef& other) const { - return &m_instructions == &other.m_instructions && m_index == other.m_index; + return m_bytes == other.m_bytes && m_index == other.m_index; } - BaseRef next() const - { - return BaseRef { m_instructions, m_index + ptr()->size() }; - } + inline Ref next() const; inline Offset offset() const { return m_index; } inline BytecodeIndex index() const { return BytecodeIndex(offset()); } bool isValid() const { - return m_index < m_instructions.size(); + return m_index < m_bytes->size(); } - private: - inline const InstructionType* unwrap() const { return reinterpret_cast(&m_instructions[m_index]); } - protected: - BaseRef(InstructionBuffer& instructions, size_t index) - : m_instructions(instructions) + BaseRef(const Bytes* bytes, size_t index) + : m_bytes(bytes) , m_index(index) { } - InstructionBuffer& m_instructions; + inline const InstructionType* unwrap() const { return reinterpret_cast(m_bytes->data() + m_index); } + + const Bytes* m_bytes; Offset m_index; }; public: - using Ref = BaseRef; + class Ref : public BaseRef { + template friend class InstructionStream; + template friend class InstructionStreamWriter; + friend class BaseRef; + protected: + using BaseRef::BaseRef; + }; - class MutableRef : public BaseRef { + class MutableRef : public BaseRef { + template friend class InstructionStream; template friend class InstructionStreamWriter; protected: - using BaseRef::BaseRef; - using BaseRef::m_index; - using BaseRef::m_instructions; + MutableRef(InstructionBuffer& buffer, const Bytes* bytes, size_t index) + : BaseRef(bytes, index) + , m_buffer(&buffer) + { } + using BaseRef::m_index; public: - Ref freeze() const { return Ref { m_instructions, m_index }; } + Ref freeze() const { return Ref { this->m_bytes, m_index }; } inline InstructionType* operator->() { return unwrap(); } - inline const InstructionType* operator->() const { return unwrap(); } + inline const InstructionType* operator->() const { return BaseRef::unwrap(); } inline InstructionType* ptr() { return unwrap(); } - inline const InstructionType* ptr() const { return unwrap(); } - inline operator Ref() - { - return Ref { m_instructions, m_index }; - } + inline const InstructionType* ptr() const { return BaseRef::unwrap(); } + inline operator Ref() const { return freeze(); } private: - inline InstructionType* unwrap() { return reinterpret_cast(&m_instructions[m_index]); } - inline const InstructionType* unwrap() const { return reinterpret_cast(&m_instructions[m_index]); } + inline InstructionType* unwrap() { return reinterpret_cast(m_buffer->mutableSpan().data() + m_index); } + + InstructionBuffer* m_buffer; }; private: @@ -165,45 +167,86 @@ class InstructionStream { public: inline iterator begin() const LIFETIME_BOUND { - return iterator { m_instructions, 0 }; + return iterator { &m_bytes, 0 }; } inline iterator end() const LIFETIME_BOUND { - return iterator { m_instructions, m_instructions.size() }; + return iterator { &m_bytes, m_bytes.size() }; } inline const Ref at(BytecodeIndex index) const { return at(index.offset()); } inline const Ref at(Offset offset) const { - ASSERT(offset < m_instructions.size()); - return Ref { m_instructions, offset }; + ASSERT(offset < m_bytes.size()); + return Ref { &m_bytes, offset }; } inline size_t size() const { - return m_instructions.size(); + return m_bytes.size(); } const void* rawPointer() const { - return m_instructions.span().data(); + return m_bytes.data(); } bool contains(InstructionType* instruction) const { auto* pointer = std::bit_cast(instruction); - return pointer >= m_instructions.begin() && pointer < m_instructions.end(); + return pointer >= m_bytes.data() && pointer < m_bytes.data() + m_bytes.size(); + } + + // Read an immutable instruction stream that lives elsewhere (inside a bytecode cache mapping) instead of copying it. + enum BorrowTag { Borrow }; + InstructionStream(Bytes borrowed, BorrowTag) + : m_bytes(borrowed) + , m_isBorrowed(true) + { } + bool isBorrowed() const { return m_isBorrowed; } + + InstructionStream(InstructionStream&& other) + : m_instructions(WTF::move(other.m_instructions)) + , m_bytes(other.m_isBorrowed ? other.m_bytes : Bytes(m_instructions.span())) + , m_isBorrowed(other.m_isBorrowed) + { + other.m_bytes = { }; + other.m_isBorrowed = false; + } + InstructionStream& operator=(InstructionStream&& other) + { + m_instructions = WTF::move(other.m_instructions); + m_isBorrowed = other.m_isBorrowed; + m_bytes = m_isBorrowed ? other.m_bytes : Bytes(m_instructions.span()); + other.m_bytes = { }; + other.m_isBorrowed = false; + return *this; } protected: explicit InstructionStream(InstructionBuffer&& instructions) : m_instructions(WTF::move(instructions)) + , m_bytes(m_instructions.span()) { } + void didMutateBuffer() + { + RELEASE_ASSERT(!m_isBorrowed); // the bytes are the cache mapping's; every write path comes through here + m_bytes = m_instructions.span(); + } + InstructionBuffer m_instructions; + Bytes m_bytes; + bool m_isBorrowed { false }; }; +template +inline typename InstructionStream::Ref InstructionStream::BaseRef::next() const +{ + return Ref { m_bytes, m_index + ptr()->size() }; +} + template class InstructionStreamWriter : public InstructionStream { friend class BytecodeRewriter; @@ -213,6 +256,8 @@ class InstructionStreamWriter : public InstructionStream { using typename InstructionStream::MutableRef; using typename InstructionStream::Offset; using InstructionStream::m_instructions; + using InstructionStream::m_bytes; + using InstructionStream::didMutateBuffer; InstructionStreamWriter() : InstructionStream({ }) @@ -223,12 +268,13 @@ class InstructionStreamWriter : public InstructionStream { RELEASE_ASSERT(!m_instructions.size()); RELEASE_ASSERT(!buffer.size()); m_instructions = WTF::move(buffer); + didMutateBuffer(); } inline MutableRef ref(Offset offset) { ASSERT(offset < m_instructions.size()); - return MutableRef { m_instructions, offset }; + return MutableRef { m_instructions, &m_bytes, offset }; } void seek(unsigned position) @@ -255,8 +301,10 @@ class InstructionStreamWriter : public InstructionStream { uint8_t* reserve() { ASSERT(!m_finalized); - if ((m_position + size) > m_instructions.size()) + if ((m_position + size) > m_instructions.size()) { m_instructions.grow(m_position + size); + didMutateBuffer(); + } auto* result = m_instructions.mutableSpan().data() + m_position; m_position += size; return result; @@ -266,6 +314,7 @@ class InstructionStreamWriter : public InstructionStream { { ASSERT(ref.offset() < m_instructions.size()); m_instructions.shrink(ref.offset()); + didMutateBuffer(); m_position = ref.offset(); } @@ -273,7 +322,9 @@ class InstructionStreamWriter : public InstructionStream { { m_finalized = true; m_instructions.shrinkToFit(); - return std::unique_ptr> { new InstructionStream(WTF::move(m_instructions)) }; + auto result = std::unique_ptr> { new InstructionStream(WTF::move(m_instructions)) }; + didMutateBuffer(); + return result; } std::unique_ptr> finalize(InstructionBuffer& usedBuffer) @@ -285,13 +336,14 @@ class InstructionStreamWriter : public InstructionStream { memcpy(resultBuffer.mutableSpan().data(), m_instructions.span().data(), m_instructions.sizeInBytes()); usedBuffer = WTF::move(m_instructions); + didMutateBuffer(); return std::unique_ptr> { new InstructionStream(WTF::move(resultBuffer)) }; } MutableRef ref() { - return MutableRef { m_instructions, m_position }; + return MutableRef { m_instructions, &m_bytes, m_position }; } void swap(InstructionStreamWriter& other) @@ -299,6 +351,8 @@ class InstructionStreamWriter : public InstructionStream { std::swap(m_finalized, other.m_finalized); std::swap(m_position, other.m_position); m_instructions.swap(other.m_instructions); + didMutateBuffer(); + other.didMutateBuffer(); } private: @@ -330,12 +384,12 @@ class InstructionStreamWriter : public InstructionStream { public: iterator begin() { - return iterator { m_instructions, 0 }; + return iterator { m_instructions, &m_bytes, 0 }; } iterator end() { - return iterator { m_instructions, m_instructions.size() }; + return iterator { m_instructions, &m_bytes, m_instructions.size() }; } private: diff --git a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp index eb9089fee154..fd4cda0904ca 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp +++ b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.cpp @@ -26,6 +26,7 @@ #include "config.h" #include "UnlinkedCodeBlock.h" +#include "HeapInlines.h" #include "BaselineJITCode.h" #include "BytecodeLivenessAnalysis.h" @@ -99,9 +100,14 @@ void UnlinkedCodeBlock::visitChildrenImpl(JSCell* cell, Visitor& visitor) UnlinkedCodeBlock* thisObject = uncheckedDowncast(cell); ASSERT_GC_OBJECT_INHERITS(thisObject, info()); Base::visitChildren(thisObject, visitor); - Locker locker { thisObject->cellLock() }; - if (visitor.isFirstVisit()) - thisObject->m_age = std::min(static_cast(thisObject->m_age) + 1, maxAge); + // Snapshot (immortal) code blocks are immutable and never jettisoned: no lock, no aging, so their pages stay clean. + bool isSnapshot = Heap::isStartupSnapshotCell(thisObject); + std::optional> locker; + if (!isSnapshot) { + locker.emplace(thisObject->cellLock()); + if (visitor.isFirstVisit()) + thisObject->m_age = std::min(static_cast(thisObject->m_age) + 1, maxAge); + } for (auto& barrier : thisObject->m_functionDecls) visitor.append(barrier); for (auto& barrier : thisObject->m_functionExprs) @@ -109,9 +115,9 @@ void UnlinkedCodeBlock::visitChildrenImpl(JSCell* cell, Visitor& visitor) visitor.appendValues(thisObject->m_constantRegisters.span()); size_t extraMemory = thisObject->metadataSizeInBytes(); if (thisObject->m_instructions) - extraMemory += thisObject->m_instructions->sizeInBytes(); + extraMemory += thisObject->m_instructions->ownedSizeInBytes(); if (thisObject->hasRareData()) - extraMemory += thisObject->m_rareData->sizeInBytes(locker); + extraMemory += locker ? thisObject->m_rareData->sizeInBytes(*locker) : thisObject->m_rareData->sizeInBytes(NoLockingNecessary); if (thisObject->m_expressionInfo) extraMemory += thisObject->m_expressionInfo->byteSize(); extraMemory += thisObject->m_jumpTargets.byteSize(); @@ -348,4 +354,22 @@ void UnlinkedCodeBlock::allocateSharedProfiles(unsigned numBinaryArithProfiles, m_unaryArithProfiles = FixedVector(numUnaryArithProfiles); } +#if USE(BUN_JSC_ADDITIONS) +UnlinkedCodeBlock::ComponentSizes UnlinkedCodeBlock::componentSizesForCensus() +{ + ComponentSizes r { }; + if (m_instructions) + r.instructions = m_instructions->sizeInBytes(); + if (m_expressionInfo) + r.expressionInfo = m_expressionInfo->byteSize(); + r.metadata = m_metadata->sizeInBytesForGC(); + r.identifiers = m_identifiers.size() * sizeof(Identifier); + r.constants = m_constantRegisters.size() * sizeof(WriteBarrier) + m_constantsSourceCodeRepresentation.size() * sizeof(SourceCodeRepresentation); + r.jumpTargets = m_jumpTargets.size() * sizeof(JSInstructionStream::Offset); + r.profiles = m_valueProfiles.size() * sizeof(UnlinkedValueProfile) + m_arrayProfiles.size() * sizeof(UnlinkedArrayProfile) + m_binaryArithProfiles.size() * sizeof(BinaryArithProfile) + m_unaryArithProfiles.size() * sizeof(UnaryArithProfile); + if (m_rareData) + r.rareData = m_rareData->sizeInBytes(NoLockingNecessary); + return r; +} +#endif } // namespace JSC diff --git a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h index d557c2125294..e57c5413e9c9 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h +++ b/Source/JavaScriptCore/bytecode/UnlinkedCodeBlock.h @@ -172,6 +172,10 @@ class UnlinkedCodeBlock : public JSCell { bool isBuiltinDefaultClassConstructor() const { return m_isBuiltinDefaultClassConstructor; } bool hasExpressionInfo() { return !m_expressionInfo->isEmpty(); } +#if USE(BUN_JSC_ADDITIONS) + struct ComponentSizes { size_t instructions { 0 }, expressionInfo { 0 }, metadata { 0 }, identifiers { 0 }, constants { 0 }, jumpTargets { 0 }, profiles { 0 }, rareData { 0 }; }; + JS_EXPORT_PRIVATE ComponentSizes componentSizesForCensus(); // memory attribution tooling +#endif bool hasCheckpoints() const { return m_hasCheckpoints; } void setHasCheckpoints() { m_hasCheckpoints = true; } @@ -324,7 +328,7 @@ class UnlinkedCodeBlock : public JSCell { static constexpr unsigned maxAge = 7; unsigned age() const { return m_age; } - void resetAge() { m_age = 0; } + void resetAge() { if (m_age) m_age = 0; } // conditional so a snapshot cell that is already young is not written NeedsClassFieldInitializer needsClassFieldInitializer() const { diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp index 7f5960c0ce39..ee483638cc2f 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp @@ -169,7 +169,7 @@ void UnlinkedFunctionExecutable::visitChildrenImpl(JSCell* cell, Visitor& visito return; if (unlinkedCodeBlock->didOptimize() == TriState::True) visitor.append(unlinkedCodeBlock); - else if (unlinkedCodeBlock->age() < UnlinkedCodeBlock::maxAge) + else if (unlinkedCodeBlock->age() < std::min(Options::unlinkedCodeBlockJettisonAge(), UnlinkedCodeBlock::maxAge)) visitor.append(unlinkedCodeBlock); }; markIfProfitable(thisObject->m_unlinkedCodeBlockForCall); @@ -239,6 +239,37 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind, OptionSet codeGenerationMode, ParserError& error, SourceParseMode parseMode) { +#if USE(BUN_JSC_ADDITIONS) + if (vm.heap.objectSpace().hasImmortalBlocks() && Heap::isStartupSnapshotCell(this)) [[unlikely]] { + // Snapshot executable: its cell is never written. Whatever it already links to (snapshotd) is used as is; anything decoded/generated now lives in the heap's side table. + if (!m_isCached) { + if (auto* codeBlock = (specializationKind == CodeSpecializationKind::CodeForCall ? m_unlinkedCodeBlockForCall : m_unlinkedCodeBlockForConstruct).get()) + return codeBlock; + } + if (auto* codeBlock = vm.heap.snapshotUnlinkedCodeBlockFor(this, specializationKind)) + return codeBlock; + WriteBarrier slot; + DeferGC deferGC(vm); + if (m_isCached) { + int32_t offset = specializationKind == CodeSpecializationKind::CodeForCall ? m_cachedCodeBlockForCallOffset : m_cachedCodeBlockForConstructOffset; + if (offset && m_decoder) { + // m_decoder is the build process's, frozen with this cell: only read from it. The decode goes through this + // process's own decoder for the same payload, and records nothing (this code was recorded when first decoded). + Ref decoder = vm.ensureBytecodeCacheDecoder(Ref { m_decoder->cachedBytecode() }, m_decoder->provider()); + decodeFunctionCodeBlockForReDecode(decoder.get(), offset, slot, this); + } + } else + decodeCodeBlockFromCacheRecord(vm, source, specializationKind, slot); + if (!slot) { + UnlinkedFunctionCodeBlock* result = generateUnlinkedFunctionCodeBlock(vm, this, source, specializationKind, codeGenerationMode, isBuiltinFunction() ? UnlinkedBuiltinFunction : UnlinkedNormalFunction, error, parseMode); + if (error.isValid()) + return nullptr; + slot.setWithoutWriteBarrier(result); + } + vm.heap.setSnapshotUnlinkedCodeBlockFor(this, specializationKind, slot.get()); + return slot.get(); + } +#endif if (m_isCached) decodeCachedCodeBlocks(vm); switch (specializationKind) { @@ -252,6 +283,9 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( break; } + if (UnlinkedFunctionCodeBlock* redecoded = tryRedecodeCodeBlock(vm, source, specializationKind)) + return redecoded; + UnlinkedFunctionCodeBlock* result = generateUnlinkedFunctionCodeBlock( vm, this, source, specializationKind, codeGenerationMode, isBuiltinFunction() ? UnlinkedBuiltinFunction : UnlinkedNormalFunction, @@ -273,6 +307,34 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( return result; } +void UnlinkedFunctionExecutable::decodeCodeBlockFromCacheRecord(VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind, WriteBarrier& slot) +{ + if (!m_isGeneratedFromCache || m_cachedRecordOffset <= 0) + return; + RefPtr provider = source.provider(); + if (!provider) + return; + RefPtr cachedBytecode = provider->cachedBytecode(); + if (!cachedBytecode || static_cast(m_cachedRecordOffset) >= cachedBytecode->size()) + return; + Ref decoder = vm.ensureBytecodeCacheDecoder(cachedBytecode.releaseNonNull(), WTF::move(provider)); + decodeFunctionCodeBlockFromExecutableRecord(decoder.get(), m_cachedRecordOffset, specializationKind, slot, *this); +} + +UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::tryRedecodeCodeBlock(VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind) +{ + if (m_isCached) + return nullptr; // still holds its decoder and offsets: the ordinary cached path applies + DeferGC deferGC(vm); + auto& slot = specializationKind == CodeSpecializationKind::CodeForCall ? m_unlinkedCodeBlockForCall : m_unlinkedCodeBlockForConstruct; + decodeCodeBlockFromCacheRecord(vm, source, specializationKind, slot); + if (!slot) + return nullptr; + vm.writeBarrier(this); + vm.heap.unlinkedFunctionExecutableSpaceAndSet.set.add(this); + return slot.get(); +} + void UnlinkedFunctionExecutable::decodeCachedCodeBlocks(VM& vm) { ASSERT(m_isCached); @@ -296,6 +358,8 @@ void UnlinkedFunctionExecutable::decodeCachedCodeBlocks(VM& vm) WTF::storeStoreFence(); m_isCached = false; vm.writeBarrier(this); + // Registered so deleteAllCode / jettisoning can clear these too; they re-decode from the cache on next use. + vm.heap.unlinkedFunctionExecutableSpaceAndSet.set.add(this); } UnlinkedFunctionExecutable::RareData& UnlinkedFunctionExecutable::ensureRareDataSlow() diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h index b59b2a846eaa..4fc21a177e63 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h @@ -138,6 +138,9 @@ class UnlinkedFunctionExecutable final : public JSCell { void recordParse(CodeFeatures features, LexicallyScopedFeatures lexicallyScopedFeatures, bool hasCapturedVariables) { + // A reparse yields the same answers; don't store (and dirty a snapshot page) unless something actually changed. + if (m_features == features && m_lexicallyScopedFeatures == lexicallyScopedFeatures && m_hasCapturedVariables == static_cast(hasCapturedVariables)) + return; m_features = features; m_lexicallyScopedFeatures = lexicallyScopedFeatures; m_hasCapturedVariables = hasCapturedVariables; @@ -281,12 +284,17 @@ class UnlinkedFunctionExecutable final : public JSCell { DECLARE_VISIT_CHILDREN; void decodeCachedCodeBlocks(VM&); + UnlinkedFunctionCodeBlock* tryRedecodeCodeBlock(VM&, const SourceCode&, CodeSpecializationKind); + void decodeCodeBlockFromCacheRecord(VM&, const SourceCode&, CodeSpecializationKind, WriteBarrier&); // caller holds a DeferGC bool codeBlockEdgeMayBeWeak() const { - // Currently, bytecode cache assumes that the tree of UnlinkedFunctionExecutable and UnlinkedCodeBlock will not be destroyed while the parent is live. - // Bytecode cache uses this asumption to avoid duplicate materialization by bookkeeping the heap cells in the offste-to-pointer map. - return VM::useUnlinkedCodeBlockJettisoning() && !m_isGeneratedFromCache; + if (!VM::useUnlinkedCodeBlockJettisoning()) + return false; + if (!m_isGeneratedFromCache) + return true; + // While m_isCached, the code block slots hold the Decoder / cache offsets rather than code blocks. + return !m_isCached && Options::useUnlinkedCodeBlockJettisoningForBytecodeCache(); } unsigned m_firstLineOffset : 31; @@ -335,6 +343,8 @@ class UnlinkedFunctionExecutable final : public JSCell { Identifier m_name; Identifier m_ecmaName; + // Offset of this executable's CachedFunctionExecutable record in the provider's CachedBytecode (0 = none); lets a cleared code block be re-decoded instead of re-parsed. + int32_t m_cachedRecordOffset { 0 }; RareData& ensureRareData() { @@ -353,7 +363,7 @@ class UnlinkedFunctionExecutable final : public JSCell { }; #if !ASSERT_ENABLED -static_assert(sizeof(UnlinkedFunctionExecutable) <= 96, "UnlinkedFunctionExecutable needs to be small"); +static_assert(sizeof(UnlinkedFunctionExecutable) <= 104, "UnlinkedFunctionExecutable needs to be small"); #endif } // namespace JSC diff --git a/Source/JavaScriptCore/bytecode/Watchpoint.cpp b/Source/JavaScriptCore/bytecode/Watchpoint.cpp index 66f84efb1fe7..57dee4c807a5 100644 --- a/Source/JavaScriptCore/bytecode/Watchpoint.cpp +++ b/Source/JavaScriptCore/bytecode/Watchpoint.cpp @@ -25,6 +25,10 @@ #include "config.h" #include "Watchpoint.h" +#if USE(BUN_JSC_ADDITIONS) +#include +#include +#endif #include "AdaptiveInferredPropertyValueWatchpointBase.h" #include "CachedSpecialPropertyAdaptiveStructureWatchpoint.h" @@ -101,6 +105,39 @@ WatchpointSet::WatchpointSet(WatchpointState state) { } + +using WatchpointList = SentinelLinkedList>; +#if USE(BUN_JSC_ADDITIONS) +// Snapshot: watchpoints added to a WatchpointSet that lives inside the snapshot go on a per-set side chain in fresh memory, +// so the push doesn't splice into (and dirty) the snapshotd neighbours/sentinel. State bytes still change in the set itself +// (JIT'd code reads them inline); the list is only ever walked here. +static Lock s_snapshotSideChainsLock; +static UncheckedKeyHashMap>& snapshotSideChains() WTF_REQUIRES_LOCK(s_snapshotSideChainsLock) +{ + static NeverDestroyed>> map; + return map; +} +static ALWAYS_INLINE bool isSnapshotWatchpointSet(WatchpointSet* set) +{ + return Heap::isInSnapshotImmortalRange(set); +} +WatchpointList* WatchpointSet::snapshotSideChain(bool createIfMissing) +{ + if (!isSnapshotWatchpointSet(this)) + return nullptr; + Locker locker { s_snapshotSideChainsLock }; + auto& map = snapshotSideChains(); + if (createIfMissing) { + auto& slot = map.add(this, nullptr).iterator->value; + if (!slot) + slot = makeUniqueWithoutFastMallocCheck(); + return slot.get(); + } + auto it = map.find(this); + return it == map.end() ? nullptr : it->value.get(); +} +#endif + WatchpointSet::~WatchpointSet() { // FIXME(rdar://165379969): This is here to silence a RefcountDebugger ASSERT. But the @@ -113,6 +150,12 @@ WatchpointSet::~WatchpointSet() // either keeping the watchpoint set's owner alive, or does some weak reference thing. while (!m_set.isEmpty()) m_set.begin()->remove(); +#if USE(BUN_JSC_ADDITIONS) + if (auto* side = snapshotSideChain(false)) { + while (!side->isEmpty()) + side->begin()->remove(); + } +#endif } void WatchpointSet::add(Watchpoint* watchpoint) @@ -121,9 +164,14 @@ void WatchpointSet::add(Watchpoint* watchpoint) ASSERT(state() != IsInvalidated); if (!watchpoint) return; - m_set.push(watchpoint); - m_setIsNotEmpty = true; - m_state = IsWatched; + if (auto* side = snapshotSideChain(true)) + side->push(watchpoint); + else + m_set.push(watchpoint); + if (!m_setIsNotEmpty) + m_setIsNotEmpty = true; + if (m_state != IsWatched) + m_state = IsWatched; } void WatchpointSet::fireAllSlow(VM& vm, const FireDetail& detail) @@ -164,8 +212,9 @@ void WatchpointSet::fireAllWatchpoints(VM& vm, const FireDetail& detail) // The safest thing to do is to DeferGCForAWhile to prevent this GC from happening. DeferGCForAWhile deferGC(vm); - while (!m_set.isEmpty()) { - Watchpoint& watchpoint = *m_set.begin(); + WatchpointList* side = snapshotSideChain(false); + while (!m_set.isEmpty() || (side && !side->isEmpty())) { + Watchpoint& watchpoint = !m_set.isEmpty() ? *m_set.begin() : *side->begin(); ASSERT(watchpoint.isOnList()); // Removing the Watchpoint before firing it makes it possible to implement watchpoints @@ -178,7 +227,6 @@ void WatchpointSet::fireAllWatchpoints(VM& vm, const FireDetail& detail) // possible to add itself to the transition watchpoint set of the singleton object's new // Structure. watchpoint.remove(); - ASSERT(&*m_set.begin() != &watchpoint); ASSERT(!watchpoint.isOnList()); watchpoint.fire(vm, detail); @@ -191,6 +239,8 @@ void WatchpointSet::take(WatchpointSet* other) { ASSERT(state() == ClearWatchpoint); m_set.takeFrom(other->m_set); + if (auto* otherSide = other->snapshotSideChain(false)) + m_set.takeFrom(*otherSide); m_setIsNotEmpty = other->m_setIsNotEmpty; m_state = other->m_state; other->m_setIsNotEmpty = false; diff --git a/Source/JavaScriptCore/bytecode/Watchpoint.h b/Source/JavaScriptCore/bytecode/Watchpoint.h index 27eaafa0dc5f..ec44c7520239 100644 --- a/Source/JavaScriptCore/bytecode/Watchpoint.h +++ b/Source/JavaScriptCore/bytecode/Watchpoint.h @@ -279,6 +279,11 @@ class WatchpointSet : public ThreadSafeRefCounted { int8_t m_setIsNotEmpty; SentinelLinkedList> m_set; +#if USE(BUN_JSC_ADDITIONS) + SentinelLinkedList>* snapshotSideChain(bool createIfMissing); // snapshot: post-restore additions to an snapshotd set +#else + static constexpr SentinelLinkedList>* snapshotSideChain(bool) { return nullptr; } +#endif }; // InlineWatchpointSet is a low-overhead, non-copyable watchpoint set in which @@ -357,7 +362,8 @@ class InlineWatchpointSet { protect(fat())->fireAll(vm, fireDetails); return; } - if (decodeState(m_data) == ClearWatchpoint) + // Already-invalidated thin sets are terminal: skip the identity store (keeps clean/shared pages clean). + if (decodeState(m_data) != IsWatched) return; m_data = encodeState(IsInvalidated); WTF::storeStoreFence(); @@ -367,7 +373,7 @@ class InlineWatchpointSet { { if (isFat()) protect(fat())->invalidate(vm, detail); - else + else if (decodeState(m_data) != IsInvalidated) m_data = encodeState(IsInvalidated); } diff --git a/Source/JavaScriptCore/heap/BlockDirectory.cpp b/Source/JavaScriptCore/heap/BlockDirectory.cpp index 93ac2023997e..3c2584268185 100644 --- a/Source/JavaScriptCore/heap/BlockDirectory.cpp +++ b/Source/JavaScriptCore/heap/BlockDirectory.cpp @@ -264,6 +264,8 @@ void BlockDirectory::lastChanceToFinalize() { forEachBlock( [&] (MarkedBlock::Handle* block) { + if (block->block().isImmortal()) [[unlikely]] + return; // snapshot memory is abandoned, never finalized: the cells belong to every launch, not to this VM block->lastChanceToFinalize(); }); } @@ -286,6 +288,11 @@ void BlockDirectory::beginMarkingForFullCollection() // as the next one is eden. markingNotEmptyBits().clearAll(); markingRetiredBits().clearAll(); +#if USE(BUN_JSC_ADDITIONS) + // Immortal blocks are never marked into, but must still be iterated by forEachMarkedCell (unconditional finalizers etc.). + markingNotEmptyBits() |= immortalBits(); + markingRetiredBits() |= immortalBits(); +#endif } void BlockDirectory::endMarking() @@ -308,8 +315,8 @@ void BlockDirectory::endMarking() // vectors. // Sweeper is suspended so we don't need the lock here. - emptyBits() = liveBits() & ~markingNotEmptyBits(); - canAllocateBits() = liveBits() & ~markingRetiredBits(); + emptyBits() = liveBits() & ~markingNotEmptyBits() & ~immortalBits(); + canAllocateBits() = liveBits() & ~markingRetiredBits() & ~immortalBits(); switch (m_attributes.destruction) { case NeedsDestruction: { @@ -342,15 +349,45 @@ void BlockDirectory::endMarking() void BlockDirectory::snapshotUnsweptForEdenCollection() { assertSweeperIsSuspended(); - unsweptBits() |= edenBits(); + unsweptBits() |= edenBits() & ~immortalBits(); } void BlockDirectory::snapshotUnsweptForFullCollection() { assertSweeperIsSuspended(); - unsweptBits() = liveBits(); + unsweptBits() = liveBits() & ~immortalBits(); } +#if USE(BUN_JSC_ADDITIONS) +void BlockDirectory::makeAllBlocksImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion) +{ + // Return any block held by an allocator — stopAllocating() records exactly the handed-out cells as newlyAllocated + // (the free-list remainder stays dead) — then make allocators forget it. + { + Locker allocatorsLocker { m_localAllocatorsLock }; + m_localAllocators.forEach([&](LocalAllocator* allocator) { + allocator->stopAllocating(); + allocator->forgetCurrentBlock(); + }); + } + Locker locker(bitvectorLock()); + assertSweeperIsSuspended(); + for (size_t index = 0; index < m_blocks.size(); ++index) { + if (!isLive(index) || !m_blocks[index]) + continue; + m_blocks[index]->block().makeImmortal(markingVersion, newlyAllocatedVersion); + setIsImmortal(index, true); + setIsMarkingNotEmpty(index, true); + setIsMarkingRetired(index, true); + setIsEmpty(index, false); + setIsCanAllocate(index, false); + setIsUnswept(index, false); + setIsEden(index, false); + setIsAllocated(index, false); + } +} +#endif + MarkedBlock::Handle* BlockDirectory::findBlockToSweep(unsigned& unsweptCursor) { Locker locker(bitvectorLock()); diff --git a/Source/JavaScriptCore/heap/BlockDirectory.h b/Source/JavaScriptCore/heap/BlockDirectory.h index a26bde2325b2..f4ed6310608e 100644 --- a/Source/JavaScriptCore/heap/BlockDirectory.h +++ b/Source/JavaScriptCore/heap/BlockDirectory.h @@ -70,6 +70,9 @@ class BlockDirectory { void endMarking(); void NODELETE snapshotUnsweptForEdenCollection(); void NODELETE snapshotUnsweptForFullCollection(); +#if USE(BUN_JSC_ADDITIONS) + void makeAllBlocksImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion); +#endif void sweep(); void shrink(); void assertNoUnswept(); diff --git a/Source/JavaScriptCore/heap/BlockDirectoryBits.h b/Source/JavaScriptCore/heap/BlockDirectoryBits.h index ab7753e77fdc..c2398ec5c5f8 100644 --- a/Source/JavaScriptCore/heap/BlockDirectoryBits.h +++ b/Source/JavaScriptCore/heap/BlockDirectoryBits.h @@ -43,6 +43,7 @@ namespace JSC { macro(eden, Eden) /* The set of all blocks that have new objects since the last GC. */\ macro(unswept, Unswept) /* The set of all blocks that could be swept by the incremental sweeper. */\ macro(inUse, InUse) /* This tells us if a block is currently being allocated from or swept. This acts like a lock bit. */\ + macro(immortal, Immortal) /* startup-snapshot blocks; only ever set under USE(BUN_JSC_ADDITIONS), so the masks below are no-ops otherwise */\ \ /* These are computed during marking. */\ macro(markingNotEmpty, MarkingNotEmpty) /* The set of all blocks that are not empty. */ \ diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index 0a4e2038f374..14991bb17f07 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -20,6 +20,9 @@ #include "config.h" #include "Heap.h" +#if USE(BUN_JSC_ADDITIONS) +#include "VerifierSlotVisitor.h" +#endif #include "JSCJSValueInlines.h" @@ -122,6 +125,9 @@ #include "JSFFICallback.h" #include "JSFFIFunction.h" #include "JSString.h" +#if USE(BUN_JSC_ADDITIONS) +#include "StructureInlines.h" +#endif #include #endif @@ -1033,6 +1039,12 @@ void Heap::endMarking() m_objectSpace.endMarking(); setMutatorShouldBeFenced(Options::forceFencedBarrier()); +#if USE(BUN_JSC_ADDITIONS) + if (m_objectSpace.hasImmortalBlocks()) { + Locker locker { m_snapshotRememberedLock }; + m_snapshotRememberedThisCycle.clear(); + } +#endif } size_t Heap::objectCount() @@ -1183,6 +1195,13 @@ void Heap::deleteAllUnlinkedCodeBlocks(DeleteAllCodeEffort effort) if (m_collectionScope && effort == DeleteAllCodeIfNotCollecting) return; +#if USE(BUN_JSC_ADDITIONS) + { + Locker locker { m_snapshotRememberedLock }; + m_snapshotUnlinkedCodeBlocks.clear(); + } +#endif + VM& vm = this->vm(); PreventCollectionScope preventCollectionScope(*this); @@ -1192,6 +1211,10 @@ void Heap::deleteAllUnlinkedCodeBlocks(DeleteAllCodeEffort effort) unlinkedFunctionExecutableSpaceAndSet.set.forEachLiveCell( [&] (HeapCell* cell, HeapCell::Kind) { UnlinkedFunctionExecutable* executable = static_cast(cell); +#if USE(BUN_JSC_ADDITIONS) + if (isStartupSnapshotCell(executable)) [[unlikely]] + return; // never written; whatever it gained after the restore was in the side table cleared above +#endif executable->clearCode(vm); }); @@ -1220,12 +1243,84 @@ void Heap::deleteUnmarkedCompiledCode() m_jitStubRoutines->deleteUnmarkedJettisonedStubRoutines(vm()); } +#if USE(BUN_JSC_ADDITIONS) +UnlinkedFunctionCodeBlock* Heap::snapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable* executable, CodeSpecializationKind kind) +{ + Locker locker { m_snapshotRememberedLock }; + auto it = m_snapshotUnlinkedCodeBlocks.find({ executable, static_cast(kind) }); + return it == m_snapshotUnlinkedCodeBlocks.end() ? nullptr : it->value; +} + +void Heap::setSnapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable* executable, CodeSpecializationKind kind, UnlinkedFunctionCodeBlock* codeBlock) +{ + Locker locker { m_snapshotRememberedLock }; + if (codeBlock) + m_snapshotUnlinkedCodeBlocks.set({ executable, static_cast(kind) }, codeBlock); + else + m_snapshotUnlinkedCodeBlocks.remove({ executable, static_cast(kind) }); +} + +void Heap::resetPacingAfterSnapshotRestore() +{ + // The limits in the snapshot were computed when the snapshot cells were ordinary live cells; here they are immortal and + // never counted, so those limits would let everything allocated after the restore accumulate in fresh blocks before + // the first collection. Start over as an empty heap does. + m_sizeAfterLastCollect = 0; + m_sizeAfterLastFullCollect = 0; + m_sizeAfterLastEdenCollect = 0; + m_extraMemorySize = 0; // reported by cells that are now immortal; a full collection would drop it too + m_deprecatedExtraMemorySize = 0; + // First cycle triggers on the mortal live size alone (immortal cells are not counted), never later than this heap type's own floor. + m_maxHeapSize = std::min(minHeapSize(m_heapType, m_ramSize), Options::mediumHeapSize()); + m_maxEdenSize = m_maxHeapSize; +} + + +void Heap::evacuateTablesForStartupSnapshot() +{ + m_objectSpace.blocks().evacuateStorage(); + rehomeOutOfSnapshotPages(m_weakGCHashTables); + { + Locker locker { m_snapshotRememberedLock }; + rehomeOutOfSnapshotPages(m_snapshotWrittenEver); + decltype(m_snapshotUnlinkedCodeBlocks) fresh; + m_snapshotUnlinkedCodeBlocks.swap(fresh); // empty at snapshot time anyway + } + if (auto* cache = vm().megamorphicCache()) + cache->age(CollectionScope::Full); // bumps the epoch (entries become misses) without rewriting the snapshotd entry arrays +} + + + +uintptr_t Heap::s_snapshotImmortalRangeLo = 0; +uintptr_t Heap::s_snapshotImmortalRangeSpan = 0; + +void Heap::rememberSnapshotCell(JSCell* cell) +{ + // Snapshot cells stay PossiblyBlack forever (their header is never written); dedupe per cycle via the side set. + Locker locker { m_snapshotRememberedLock }; + m_snapshotWrittenEver.add(cell); + if (m_snapshotRememberedThisCycle.add(cell).isNewEntry) + m_mutatorMarkStack->append(cell); +} + +void Heap::willVisitSnapshotCell(JSCell* cell) +{ + Locker locker { m_snapshotRememberedLock }; + m_snapshotRememberedThisCycle.remove(cell); +} +#endif + void Heap::addToRememberedSet(const JSCell* constCell) { JSCell* cell = const_cast(constCell); ASSERT(cell); ASSERT(!Options::useConcurrentJIT() || !isCompilationThread()); m_barriersExecuted++; + if (m_objectSpace.hasImmortalBlocks() && isStartupSnapshotCell(cell)) [[unlikely]] { + rememberSnapshotCell(cell); + return; + } if (m_mutatorShouldBeFenced) { WTF::loadLoadFence(); if (!isMarked(cell)) { @@ -3064,8 +3159,167 @@ constexpr bool samplingProfilerSupported = false; static UNUSED_FUNCTION void visitSamplingProfiler(VM&, AbstractSlotVisitor&) { }; #endif +#if USE(BUN_JSC_ADDITIONS) +void Heap::freezeCurrentHeapAsImmortalStartupSnapshot() +{ + RELEASE_ASSERT(vm().currentThreadIsHoldingAPILock()); + vm().completeAllJITPlansBeforeStartupSnapshot(); // no compiler thread may be reading structures while prepareForStartupSnapshot() switches their lock + { + // Settle lazily-materialized state that would otherwise be written into snapshot cells after freeze: + // every Structure gets its property table now (and the freeze GC below keeps them). + Vector structures; + { + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (isJSCellKind(kind) && static_cast(heapCell)->type() == StructureType) + structures.append(static_cast(static_cast(heapCell))); + return IterationStatus::Continue; + }); + } // materializing tables allocates, which iteration forbids + DeferGC deferGC(vm()); + size_t withTable = 0; + for (Structure* structure : structures) { + structure->prepareForStartupSnapshot(vm()); + if (structure->hasPropertyTableForSnapshot()) + withTable++; + } + dataLogLnIf(Options::verboseStartupSnapshotFreeze(), "[imm] pre-materialized property tables: ", withTable, " of ", structures.size(), " structures have a table"); + } + collectNow(Sync, CollectionScope::Full); + if (Options::verboseStartupSnapshotFreeze()) [[unlikely]] { + size_t withTable = 0, total = 0; + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (isJSCellKind(kind) && static_cast(heapCell)->type() == StructureType) { + total++; + if (static_cast(static_cast(heapCell))->hasPropertyTableForSnapshot()) + withTable++; + } + return IterationStatus::Continue; + }); + dataLogLn("[imm] after freeze GC: ", withTable, " of ", total, " structures have a table"); + } + PreventCollectionScope preventCollectionScope(*this); + sweeper().stopSweeping(); // nothing sweeps immortal blocks; a pending incremental sweep would rewrite their pages + m_objectSpace.freezeAllBlocksAsImmortal(); + m_objectSpace.forEachSubspace([](Subspace& subspace) { + if (subspace.isIsoSubspace()) + static_cast(subspace).abandonLowerTierPreciseFreeListForSnapshot(); + return IterationStatus::Continue; + }); + for (PreciseAllocation* allocation : m_objectSpace.preciseAllocations()) { + if (!allocation->isLive()) + continue; + allocation->makeImmortal(); + if (isJSCellKind(allocation->attributes().cellKind)) { + JSCell* cell = static_cast(allocation->cell()); + m_snapshotPreciseRoots.append(cell); + if (cell->cellState() != CellState::PossiblyBlack) + cell->setCellState(CellState::PossiblyBlack); // so stores into it take the barrier slow path (-> side remembered set) like every other snapshot cell + } + } + { + // Cells still PossiblyGrey (remembered around the freeze GC) would get blackened on their next visit; settle them now + // (they stay on the mutator mark stack, so nothing is lost) so snapshot headers are never written afterwards. + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (isJSCellKind(kind) && !heapCell->isPreciseAllocation() && heapCell->markedBlock().isImmortal()) { + JSCell* cell = static_cast(heapCell); + if (cell->cellState() != CellState::PossiblyBlack) { + rememberSnapshotCell(cell); + cell->setCellState(CellState::PossiblyBlack); + } + } + return IterationStatus::Continue; + }); + } + if (Options::verboseStartupSnapshotFreeze()) [[unlikely]] { + size_t states[4] = { 0, 0, 0, 0 }; + HashMap whiteByClass; + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (!isJSCellKind(kind) || heapCell->isPreciseAllocation() || !heapCell->markedBlock().isImmortal()) + return IterationStatus::Continue; + auto st = static_cast(static_cast(heapCell)->cellState()); + states[std::min(st, 3u)]++; + if (st == static_cast(CellState::DefinitelyWhite)) + whiteByClass.add(static_cast(heapCell)->className(), 0).iterator->value++; + return IterationStatus::Continue; + }); + dataLogLn("[imm] frozen cellState histogram: PossiblyBlack=", states[0], " DefinitelyWhite=", states[1], " PossiblyGrey=", states[2], " other=", states[3]); + for (auto& e : whiteByClass) + dataLogLn(" white: ", e.key, " x", e.value); + } + { + size_t count = 0, atoms = 0; + HeapIterationScope iterationScope(*this); + m_objectSpace.forEachLiveCell(iterationScope, [&](HeapCell* heapCell, HeapCell::Kind kind) { + if (!isJSCellKind(kind)) + return IterationStatus::Continue; + JSCell* cell = static_cast(heapCell); + if (cell->isString()) { + if (auto* impl = static_cast(cell)->tryGetValueImpl()) { + impl->settleLazyHeaderWritesForStartupSnapshot(); + impl->makeStaticForSnapshot(); + count++; + } + } + return IterationStatus::Continue; + }); + for (auto& entry : vm().atomStringTable()->table()) { + if (StringImpl* impl = entry.get()) { + impl->settleLazyHeaderWritesForStartupSnapshot(); + impl->makeStaticForSnapshot(); + atoms++; + } + } + dataLogLnIf(Options::verboseStartupSnapshotFreeze(), "[imm] made ", count, " string cells' StringImpls static, and ", atoms, " atom-table entries (overlapping sets)"); + } +} +#endif + void Heap::addCoreConstraints() { +#if USE(BUN_JSC_ADDITIONS) + m_constraintSet->add( + "Img", "Immortal snapshot roots", + MAKE_MARKING_CONSTRAINT_EXECUTOR_PAIR(([this] (auto& visitor) { + if (!m_objectSpace.hasImmortalBlocks()) + return; + Vector redecoded; + { + Locker locker { m_snapshotRememberedLock }; + redecoded = copyToVector(m_snapshotUnlinkedCodeBlocks.values()); + } + for (auto* codeBlock : redecoded) + visitor.appendUnbarriered(JSValue(static_cast(codeBlock))); + // Unwritten snapshot cells can only reference snapshot cells or the precise allocations that existed at freeze; + // so the Full-GC roots are: every snapshot cell written since freeze (side card set) + those precise allocations. + if (m_collectionScope != CollectionScope::Full) + return; + SetRootMarkReasonScope rootScope(visitor, RootMarkReason::StrongReferences); + Vector written; + { + Locker locker { m_snapshotRememberedLock }; + written = copyToVector(m_snapshotWrittenEver); + } + for (JSCell* cell : written) { + if constexpr (std::is_same_v, SlotVisitor>) + visitor.visitImmortalCellAsRoot(cell); + else + visitor.appendUnbarriered(JSValue(cell)); + } + for (JSCell* cell : m_snapshotPreciseRoots) { // immortal too: appendUnbarriered would see "already marked" and never look inside + if constexpr (std::is_same_v, SlotVisitor>) + visitor.visitImmortalCellAsRoot(cell); + else + visitor.appendUnbarriered(JSValue(cell)); + } + })), + ConstraintVolatility::GreyedByExecution); +#endif + + m_constraintSet->add( "Cs", "Conservative Scan", MAKE_MARKING_CONSTRAINT_EXECUTOR_PAIR(([this, lastVersion = static_cast(0)] (auto& visitor) mutable { @@ -3505,7 +3759,10 @@ void Heap::verifyGC() if (Heap::isMarked(cell)) return; - dataLogLn("\n" "GC Verifier: ERROR cell ", RawPointer(cell), " was not marked"); + dataLogLn("\n" "GC Verifier: ERROR cell ", RawPointer(cell), " was not marked", + " type=", isJSCellKind(cell->cellKind()) ? static_cast(cell)->className() : "aux", + " immortalBlock=", (!cell->isPreciseAllocation() && cell->markedBlock().isImmortal()), + " cellState=", isJSCellKind(cell->cellKind()) ? static_cast(static_cast(cell)->cellState()) : -1); if (Options::verboseVerifyGC()) [[unlikely]] visitor.dumpMarkerData(cell); RELEASE_ASSERT(this->isMarked(cell)); diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h index bcf1df069af1..8ac47c0c7405 100644 --- a/Source/JavaScriptCore/heap/Heap.h +++ b/Source/JavaScriptCore/heap/Heap.h @@ -23,6 +23,7 @@ #include "ArrayBuffer.h" #include "CellState.h" +#include "CodeSpecializationKind.h" #include "CollectionScope.h" #include "CollectorPhase.h" #include "CompleteSubspace.h" @@ -104,6 +105,11 @@ class SpaceTimeMutatorScheduler; class StopIfNecessaryTimer; class SweepingScope; class VM; +class StructureTransitionTable; +class StructureChain; +class Structure; +class UnlinkedFunctionExecutable; +class UnlinkedFunctionCodeBlock; class VerifierSlotVisitor; class WeakGCHashTable; struct CurrentThreadState; @@ -478,6 +484,42 @@ class Heap { JS_EXPORT_PRIVATE bool unprotect(JSValue); // True when the protect count drops to 0. JS_EXPORT_PRIVATE size_t extraMemorySize(); // Non-GC memory referenced by GC objects. +#if USE(BUN_JSC_ADDITIONS) + // Snapshot: after a synchronous full collection, freeze every MarkedBlock as an immortal snapshot block. + JS_EXPORT_PRIVATE void freezeCurrentHeapAsImmortalStartupSnapshot(); + static inline bool isStartupSnapshotCell(const JSCell*); + // Rule 2 (link state out of snapshot cells): decoded UnlinkedFunctionCodeBlocks for snapshot UnlinkedFunctionExecutables live here, not in the (never written) cell. + UnlinkedFunctionCodeBlock* snapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind); + void setSnapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind, UnlinkedFunctionCodeBlock*); + void clearSnapshotUnlinkedCodeBlocks() + { + Locker locker { m_snapshotRememberedLock }; + m_snapshotUnlinkedCodeBlocks.clear(); + } + // A table that lives on in the restored process gets fresh backing storage: growing it in place would dirty snapshot pages. + template static void rehomeOutOfSnapshotPages(Table& table) + { + Table copy = table; + table.swap(copy); + } + void evacuateTablesForStartupSnapshot(); // after restore: re-home per-heap tables that take inserts every run + JS_EXPORT_PRIVATE void resetPacingAfterSnapshotRestore(); + // Barrier slow path for snapshot owners: side remembered set instead of cellState writes. + void rememberSnapshotCell(JSCell*); + // [lo, lo+span) is the mapped snapshot's allocator range (span 0: none); the embedder sets it at restore. Slow paths only. + JS_EXPORT_PRIVATE static uintptr_t s_snapshotImmortalRangeLo; + JS_EXPORT_PRIVATE static uintptr_t s_snapshotImmortalRangeSpan; + static bool isInSnapshotImmortalRange(const void* p) { return reinterpret_cast(p) - s_snapshotImmortalRangeLo < s_snapshotImmortalRangeSpan; } + // Called by the visitor right before it reads a snapshot cell's fields, so a racing store re-remembers it. + void willVisitSnapshotCell(JSCell*); +#else + static bool isStartupSnapshotCell(const JSCell*) { return false; } + static bool isInSnapshotImmortalRange(const void*) { return false; } + UnlinkedFunctionCodeBlock* snapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind) { return nullptr; } + void setSnapshotUnlinkedCodeBlockFor(const UnlinkedFunctionExecutable*, CodeSpecializationKind, UnlinkedFunctionCodeBlock*) { } + void rememberSnapshotCell(JSCell*) { } + void willVisitSnapshotCell(JSCell*) { } +#endif JS_EXPORT_PRIVATE size_t size(); JS_EXPORT_PRIVATE size_t capacity(); JS_EXPORT_PRIVATE size_t objectCount(); @@ -902,6 +944,14 @@ class Heap { std::unique_ptr m_collectorSlotVisitor; std::unique_ptr m_mutatorSlotVisitor; std::unique_ptr m_mutatorMarkStack; +#if USE(BUN_JSC_ADDITIONS) + // Prototype card table: snapshot cells written since freeze (sticky, Full-GC roots) and this cycle (barrier dedupe). + Lock m_snapshotRememberedLock; + UncheckedKeyHashSet m_snapshotWrittenEver WTF_GUARDED_BY_LOCK(m_snapshotRememberedLock); + UncheckedKeyHashMap, UnlinkedFunctionCodeBlock*> m_snapshotUnlinkedCodeBlocks WTF_GUARDED_BY_LOCK(m_snapshotRememberedLock); // visited as roots by the Img constraint + UncheckedKeyHashSet m_snapshotRememberedThisCycle WTF_GUARDED_BY_LOCK(m_snapshotRememberedLock); + Vector m_snapshotPreciseRoots; // filled once at freeze with the world stopped, read-only afterwards: no lock +#endif std::unique_ptr m_raceMarkStack; std::unique_ptr m_constraintSet; std::unique_ptr m_verifierSlotVisitor; diff --git a/Source/JavaScriptCore/heap/HeapInlines.h b/Source/JavaScriptCore/heap/HeapInlines.h index 9c6647adfee2..cace8a5b8709 100644 --- a/Source/JavaScriptCore/heap/HeapInlines.h +++ b/Source/JavaScriptCore/heap/HeapInlines.h @@ -56,6 +56,17 @@ inline JSC::Heap* Heap::heap(const JSValue v) return heap(v.asCell()); } +#if USE(BUN_JSC_ADDITIONS) +ALWAYS_INLINE bool Heap::isStartupSnapshotCell(const JSCell* cell) +{ + if (!cell) + return false; + if (cell->isPreciseAllocation()) + return cell->preciseAllocation().isImmortal(); + return cell->markedBlock().isImmortal(); +} +#endif + // Defined in Heap.cpp with NEVER_INLINE to prevent LTO from breaking compiler barriers // ALWAYS_INLINE bool Heap::isMarked(const void* rawCell) // { diff --git a/Source/JavaScriptCore/heap/IsoSubspace.h b/Source/JavaScriptCore/heap/IsoSubspace.h index b5e9bd58ee49..b144bc3f66ec 100644 --- a/Source/JavaScriptCore/heap/IsoSubspace.h +++ b/Source/JavaScriptCore/heap/IsoSubspace.h @@ -54,6 +54,14 @@ class IsoSubspace final : public Subspace { void* tryAllocateLowerTierPrecise(size_t cellSize); void destroyLowerTierPreciseFreeList(); +#if USE(BUN_JSC_ADDITIONS) + // The free cells live in snapshot pages: never hand them out again. + void abandonLowerTierPreciseFreeListForSnapshot() + { + while (!m_lowerTierPreciseFreeList.isEmpty()) + m_lowerTierPreciseFreeList.remove(&*m_lowerTierPreciseFreeList.begin()); + } +#endif void sweep(); diff --git a/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp b/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp index 98ce80022f30..35e6274daa68 100644 --- a/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp +++ b/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp @@ -95,8 +95,10 @@ void JITStubRoutineSet::prepareForConservativeScan() void JITStubRoutineSet::clearMarks() { // Immutable code routines do not matter. - for (auto& entry : m_routines) - entry.routine->m_mayBeExecuting = false; + for (auto& entry : m_routines) { + if (entry.routine->m_mayBeExecuting) + entry.routine->m_mayBeExecuting = false; + } } void JITStubRoutineSet::markSlow(uintptr_t address) @@ -135,8 +137,8 @@ void JITStubRoutineSet::deleteUnmarkedJettisonedStubRoutines(VM& vm) ASSERT(vm.heap.isInPhase(CollectorPhase::End)); auto shouldRemove = [&](GCAwareJITStubRoutine* stub) { - if (!stub->m_ownerIsDead) - stub->m_ownerIsDead = stub->removeDeadOwners(vm); + if (!stub->m_ownerIsDead && stub->removeDeadOwners(vm)) + stub->m_ownerIsDead = true; // If the stub is running right now, we should keep it alive regardless of whether owner CodeBlock gets dead. // It is OK since we already marked all the related cells. diff --git a/Source/JavaScriptCore/heap/LocalAllocator.h b/Source/JavaScriptCore/heap/LocalAllocator.h index 37727a06d827..054ab0581262 100644 --- a/Source/JavaScriptCore/heap/LocalAllocator.h +++ b/Source/JavaScriptCore/heap/LocalAllocator.h @@ -51,6 +51,9 @@ class LocalAllocator : public BasicRawSentinelNode { void prepareForAllocation(); void resumeAllocating(); void stopAllocatingForGood(); +#if USE(BUN_JSC_ADDITIONS) + void forgetCurrentBlock() { m_freeList.clear(); m_currentBlock = nullptr; m_lastActiveBlock = nullptr; } +#endif static constexpr ptrdiff_t offsetOfFreeList(); static constexpr ptrdiff_t offsetOfCellSize(); diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp index 4430cbd053e2..94d7094d9aec 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.cpp +++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp @@ -47,6 +47,8 @@ namespace JSC { // compiler barriers (loadLoadFence/compilerFence) on x86_64. NEVER_INLINE bool MarkedBlock::isMarked(HeapVersion markingVersion, const void* p) { + if (isImmortal()) [[unlikely]] + return header().m_marks.get(atomNumber(p)); HeapVersion version; Dependency dependency = Dependency::loadAndFence(&header().m_markingVersion, version); if (version != markingVersion) [[unlikely]] @@ -54,10 +56,28 @@ NEVER_INLINE bool MarkedBlock::isMarked(HeapVersion markingVersion, const void* return header().m_marks.concurrentGet(atomNumber(p), dependency); } +#if USE(BUN_JSC_ADDITIONS) +void MarkedBlock::makeImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion) +{ + // Runs right after a synchronous full GC + sweep: m_marks (if current) ∪ newlyAllocated (if current) is exactly the + // live set; every other slot is a zapped free cell that nothing references. That bitmap is frozen here and never + // written again, so the stock mark fast paths are safe on snapshot blocks: a live snapshot cell reads "already marked" + // (write-free), and a dead slot is unreachable by precise marking (conservative roots filter through isLive()). + if (header().m_markingVersion != markingVersion) + header().m_marks.clearAll(); + if (header().m_newlyAllocatedVersion == newlyAllocatedVersion) + header().m_marks.merge(header().m_newlyAllocated); + header().m_markingVersion = markingVersion; + header().m_isImmortal = true; +} +#endif + // NEVER_INLINE to prevent LTO from inlining this function, which can break // compiler barriers (Dependency::fence/loadLoadFence/compilerFence) on x86_64. NEVER_INLINE bool MarkedBlock::Handle::isLive(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion, bool isMarking, const HeapCell* cell) { + if (block().isImmortal()) [[unlikely]] + return block().isMarkedRaw(cell); m_directory->assertIsMutatorOrMutatorIsStopped(); if (m_directory->isAllocated(this)) return true; @@ -333,6 +353,8 @@ inline void MarkedBlock::setupTestForDumpInfoAndCrash() { } void MarkedBlock::aboutToMarkSlow(HeapVersion markingVersion, HeapCell* cell) { + if (isImmortal()) [[unlikely]] + return; ASSERT(vm().heap.objectSpace().isMarking()); setupTestForDumpInfoAndCrash(); @@ -420,7 +442,7 @@ void MarkedBlock::resetMarks() #if ASSERT_ENABLED void MarkedBlock::assertMarksNotStale() { - ASSERT(header().m_markingVersion == vm().heap.objectSpace().markingVersion()); + ASSERT(isImmortal() || header().m_markingVersion == vm().heap.objectSpace().markingVersion()); } #endif // ASSERT_ENABLED @@ -548,6 +570,10 @@ Subspace* MarkedBlock::Handle::subspace() const void MarkedBlock::Handle::sweep(FreeList* freeList) { + if (block().isImmortal()) [[unlikely]] { + dataLogLn("[imm] BUG: sweeping immortal block ", RawPointer(&block()), " toFreeList=", !!freeList, " dir empty=", m_directory->isEmpty(this), " canAlloc=", m_directory->isCanAllocate(this), " unswept=", m_directory->isUnswept(this), " immortalBit=", m_directory->isImmortal(this)); + CRASH(); + } SweepingScope sweepingScope(*heap()); m_directory->assertIsMutatorOrMutatorIsStopped(); ASSERT(m_directory->isInUse(this)); diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h index 1bf8d98a2bae..b108b660690d 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.h +++ b/Source/JavaScriptCore/heap/MarkedBlock.h @@ -313,6 +313,10 @@ class MarkedBlock { WTF::BitSet m_marks; WTF::BitSet m_newlyAllocated; void* m_verifierMemo { nullptr }; +#if USE(BUN_JSC_ADDITIONS) + // Immortal (snapshot) block: liveness is frozen in m_marks, the collector never writes here, never sweeps or allocates. + bool m_isImmortal { false }; +#endif }; private: @@ -403,6 +407,12 @@ class MarkedBlock { bool isMarkedRaw(const void* p); HeapVersion markingVersion() const { return header().m_markingVersion; } +#if USE(BUN_JSC_ADDITIONS) + bool isImmortal() const { return header().m_isImmortal; } + void makeImmortal(HeapVersion markingVersion, HeapVersion newlyAllocatedVersion); +#else + constexpr bool isImmortal() const { return false; } +#endif const WTF::BitSet& marks() const; @@ -584,6 +594,8 @@ inline unsigned MarkedBlock::atomNumber(const void* p) inline bool MarkedBlock::areMarksStale(HeapVersion markingVersion) { + if (isImmortal()) [[unlikely]] + return false; // a snapshot block keeps the version it was frozen with; its marks are its liveness return markingVersion != header().m_markingVersion; } @@ -592,7 +604,7 @@ inline Dependency MarkedBlock::aboutToMark(HeapVersion markingVersion, HeapCell* HeapVersion version; Dependency dependency = Dependency::loadAndFence(&header().m_markingVersion, version); if (version != markingVersion) [[unlikely]] - aboutToMarkSlow(markingVersion, cell); + aboutToMarkSlow(markingVersion, cell); // snapshot blocks keep a stale version forever; the slow path returns without writing return dependency; } @@ -624,6 +636,9 @@ inline bool MarkedBlock::isMarked(const void* p, Dependency dependency) inline bool MarkedBlock::testAndSetMarked(const void* p, Dependency dependency) { + // Snapshot blocks need no case here: every snapshot cell a precise pointer can reach is live, so its frozen bit is set and + // concurrentTestAndSet returns without writing; dead snapshot cells are only ever proposed by conservative scanning, + // which consults isMarked (areMarksStale) first. assertMarksNotStale(); return header().m_marks.concurrentTestAndSet(atomNumber(p), dependency); } diff --git a/Source/JavaScriptCore/heap/MarkedBlockSet.h b/Source/JavaScriptCore/heap/MarkedBlockSet.h index a75958e41066..943fccf37068 100644 --- a/Source/JavaScriptCore/heap/MarkedBlockSet.h +++ b/Source/JavaScriptCore/heap/MarkedBlockSet.h @@ -40,6 +40,9 @@ class MarkedBlockSet { TinyBloomFilter filter() const; const UncheckedKeyHashSet& set() const; +#if USE(BUN_JSC_ADDITIONS) + void evacuateStorage() { auto copy = m_set; m_set.swap(copy); } // snapshot: move the table to fresh pages so inserts don't dirty snapshotd ones +#endif private: void recomputeFilter(); diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp index 369ca6660700..9ebab02dd637 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.cpp +++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp @@ -265,7 +265,8 @@ void MarkedSpace::sweepPreciseAllocations() } continue; } - allocation->setIndexInSpace(dstIndex); + if (allocation->indexInSpace() != dstIndex) // snapshot allocations form a stable prefix: no store to their headers + allocation->setIndexInSpace(dstIndex); m_preciseAllocations[dstIndex++] = allocation; } m_preciseAllocations.shrinkCapacity(dstIndex); @@ -344,7 +345,8 @@ void MarkedSpace::prepareForConservativeScan() }); unsigned index = m_preciseAllocationsOffsetForThisCollection; for (auto* start = m_preciseAllocationsForThisCollectionBegin; start != m_preciseAllocationsForThisCollectionEnd; ++start, ++index) { - (*start)->setIndexInSpace(index); + if ((*start)->indexInSpace() != index) + (*start)->setIndexInSpace(index); ASSERT(m_preciseAllocations[index] == *start); ASSERT(m_preciseAllocations[index]->indexInSpace() == index); } @@ -399,6 +401,19 @@ MarkedBlock::Handle* MarkedSpace::findMarkedBlockHandleDebug(MarkedBlock* block) return result; } +#if USE(BUN_JSC_ADDITIONS) +void MarkedSpace::freezeAllBlocksAsImmortal() +{ + HeapVersion newlyAllocatedVersion = this->newlyAllocatedVersion(); + HeapVersion markingVersion = this->markingVersion(); + forEachDirectory([&](BlockDirectory& directory) -> IterationStatus { + directory.makeAllBlocksImmortal(markingVersion, newlyAllocatedVersion); + return IterationStatus::Continue; + }); + m_hasImmortalBlocks = true; +} +#endif + void MarkedSpace::freeBlock(MarkedBlock::Handle* block) { m_capacity -= MarkedBlock::blockSize; @@ -468,8 +483,10 @@ void MarkedSpace::endMarking() m_newlyAllocatedVersion = nextVersion(m_newlyAllocatedVersion); - for (unsigned i = m_preciseAllocationsOffsetForThisCollection; i < m_preciseAllocations.size(); ++i) - m_preciseAllocations[i]->clearNewlyAllocated(); + for (unsigned i = m_preciseAllocationsOffsetForThisCollection; i < m_preciseAllocations.size(); ++i) { + if (m_preciseAllocations[i]->isNewlyAllocated()) // same: an unconditional store would dirty every snapshot allocation's header + m_preciseAllocations[i]->clearNewlyAllocated(); + } if (ASSERT_ENABLED) { for (PreciseAllocation* allocation : m_preciseAllocations) diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h index 1c22af51cddc..4277aa5161e7 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.h +++ b/Source/JavaScriptCore/heap/MarkedSpace.h @@ -127,6 +127,13 @@ class MarkedSpace { template void forEachLiveCell(HeapIterationScope&, const Functor&); template void forEachDeadCell(HeapIterationScope&, const Functor&); template void forEachBlock(const Functor&); +#if USE(BUN_JSC_ADDITIONS) + // Snapshot: freeze every current block as an immortal snapshot block (call with the world stopped, right after a Full GC). + void freezeAllBlocksAsImmortal(); + bool hasImmortalBlocks() const { return m_hasImmortalBlocks; } +#else + constexpr bool hasImmortalBlocks() const { return false; } +#endif template void forEachSubspace(const Functor&); void shrink(); @@ -218,6 +225,9 @@ class MarkedSpace { HeapVersion m_newlyAllocatedVersion { initialVersion }; HeapVersion m_edenVersion { initialVersion }; bool m_isIterating { false }; +#if USE(BUN_JSC_ADDITIONS) + bool m_hasImmortalBlocks { false }; +#endif bool m_isMarking { false }; bool m_conservativeScanIsPrepared { false }; Lock m_directoryLock; diff --git a/Source/JavaScriptCore/heap/PreciseAllocation.cpp b/Source/JavaScriptCore/heap/PreciseAllocation.cpp index f439bd312ddb..12347188976a 100644 --- a/Source/JavaScriptCore/heap/PreciseAllocation.cpp +++ b/Source/JavaScriptCore/heap/PreciseAllocation.cpp @@ -122,6 +122,9 @@ PreciseAllocation* PreciseAllocation::tryReallocate(size_t size, Subspace* subsp unsigned oldAdjustment = m_adjustment; void* oldBasePointer = basePointer(); +#if USE(BUN_JSC_ADDITIONS) + ASSERT(!isImmortal()); // ButterflyInlines never reallocates a snapshot allocation +#endif void* newSpace = subspace->alignedMemoryAllocator()->tryReallocateMemory(oldBasePointer, adjustedAlignmentAllocationSize); if (!newSpace) return nullptr; @@ -232,6 +235,8 @@ PreciseAllocation::~PreciseAllocation() void PreciseAllocation::lastChanceToFinalize() { + if (isImmortal()) [[unlikely]] + return; // as for immortal blocks: abandoned, not finalized m_weakSet.lastChanceToFinalize(); clearMarked(); clearNewlyAllocated(); @@ -259,6 +264,8 @@ void PreciseAllocation::flip() // The dead object newly created before this 1 0 => 1 0 => 1 0 => 0 0 => dead // ^ // This is ensured since this function is used only for full GC. + if (isImmortal()) + return; m_isNewlyAllocated |= isMarked(); m_isMarked.store(false, std::memory_order_relaxed); } @@ -271,6 +278,8 @@ bool PreciseAllocation::isEmpty() void PreciseAllocation::sweep() { m_weakSet.sweep(); + if (isImmortal()) + return; if (m_hasValidCell && !isLive()) { if (m_attributes.destruction != DoesNotNeedDestruction) diff --git a/Source/JavaScriptCore/heap/PreciseAllocation.h b/Source/JavaScriptCore/heap/PreciseAllocation.h index a6eee51bada3..3f7003304771 100644 --- a/Source/JavaScriptCore/heap/PreciseAllocation.h +++ b/Source/JavaScriptCore/heap/PreciseAllocation.h @@ -87,7 +87,13 @@ class PreciseAllocation : public BasicRawSentinelNode { void NODELETE flip(); bool isNewlyAllocated() const { return m_isNewlyAllocated; } - ALWAYS_INLINE bool isMarked() { return m_isMarked.load(std::memory_order_relaxed); } + ALWAYS_INLINE bool isMarked() { return isImmortal() || m_isMarked.load(std::memory_order_relaxed); } +#if USE(BUN_JSC_ADDITIONS) + void makeImmortal() { m_isImmortal = true; } + bool isImmortal() const { return m_isImmortal; } +#else + constexpr bool isImmortal() const { return false; } +#endif ALWAYS_INLINE bool isMarked(HeapCell*) { return isMarked(); } ALWAYS_INLINE bool isMarked(HeapCell*, Dependency) { return isMarked(); } ALWAYS_INLINE bool isMarked(HeapVersion, HeapCell*) { return isMarked(); } @@ -173,6 +179,9 @@ class PreciseAllocation : public BasicRawSentinelNode { size_t m_cellSize; bool m_isNewlyAllocated : 1; bool m_hasValidCell : 1; +#if USE(BUN_JSC_ADDITIONS) + bool m_isImmortal : 1 { false }; +#endif // Worst case adjustment needed would be halfAlignment + portionOfObjectThatMustFitInCacheLine // which is 8 + 16 -> 24 bytes i.e. will fit in 5 bits. If we need more bits in the future, we // can also encode this number of uintptr_t words to save 3 bits. diff --git a/Source/JavaScriptCore/heap/SlotVisitor.cpp b/Source/JavaScriptCore/heap/SlotVisitor.cpp index 75082f85b03a..fa4162b71ed3 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.cpp +++ b/Source/JavaScriptCore/heap/SlotVisitor.cpp @@ -344,6 +344,18 @@ class SetCurrentCellScope { SlotVisitor& m_visitor; }; +#if USE(BUN_JSC_ADDITIONS) +void SlotVisitor::visitImmortalCellAsRoot(const JSCell* cell) +{ + // Like visitChildren, but never writes the (frozen, already-black) header, so clean snapshot pages stay clean. + SetCurrentCellScope currentCellScope(*this, cell); + m_isFirstVisit = false; + m_heap.willVisitSnapshotCell(const_cast(cell)); + WTF::storeLoadFence(); + cell->methodTable()->visitChildren(const_cast(cell), *this); +} +#endif + ALWAYS_INLINE void SlotVisitor::visitChildren(const JSCell* cell) { ASSERT(m_heap.isMarked(cell)); @@ -362,7 +374,10 @@ ALWAYS_INLINE void SlotVisitor::visitChildren(const JSCell* cell) // not clear to me that it would be correct or profitable to bail here if the object is already // black. - cell->setCellState(CellState::PossiblyBlack); + if (cell->cellState() != CellState::PossiblyBlack) + cell->setCellState(CellState::PossiblyBlack); + else if (m_heap.objectSpace().hasImmortalBlocks() && Heap::isStartupSnapshotCell(cell)) [[unlikely]] + m_heap.willVisitSnapshotCell(const_cast(cell)); WTF::storeLoadFence(); diff --git a/Source/JavaScriptCore/heap/SlotVisitor.h b/Source/JavaScriptCore/heap/SlotVisitor.h index e34194d26666..943391cbb3b6 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.h +++ b/Source/JavaScriptCore/heap/SlotVisitor.h @@ -208,6 +208,9 @@ class SlotVisitor final : public AbstractSlotVisitor { void noteLiveAuxiliaryCell(HeapCell*); void visitChildren(const JSCell*); +#if USE(BUN_JSC_ADDITIONS) + void visitImmortalCellAsRoot(const JSCell*); +#endif void propagateExternalMemoryVisitedIfNecessary(); diff --git a/Source/JavaScriptCore/heap/WeakSet.h b/Source/JavaScriptCore/heap/WeakSet.h index 033fd4d45fca..95d6b2fb62b2 100644 --- a/Source/JavaScriptCore/heap/WeakSet.h +++ b/Source/JavaScriptCore/heap/WeakSet.h @@ -122,8 +122,12 @@ ALWAYS_INLINE void WeakSet::deallocate(WeakImpl* weakImpl) inline void WeakSet::resetAllocator() { - m_allocator = nullptr; - m_nextAllocator = m_blocks.head(); + // Compare first: this runs for every weak set on every full collection, including the snapshot cells' whose values are + // already these, and a store would dirty their pages for nothing. + if (m_allocator) + m_allocator = nullptr; + if (WeakBlock* head = m_blocks.head(); m_nextAllocator != head) + m_nextAllocator = head; } } // namespace JSC diff --git a/Source/JavaScriptCore/runtime/ButterflyInlines.h b/Source/JavaScriptCore/runtime/ButterflyInlines.h index d3f4333c2a4a..d3cbe750bea2 100644 --- a/Source/JavaScriptCore/runtime/ButterflyInlines.h +++ b/Source/JavaScriptCore/runtime/ButterflyInlines.h @@ -205,7 +205,9 @@ inline Butterfly* Butterfly::reallocArrayRightIfPossible( // We can eagerly destroy butterfly backed by PreciseAllocation if (1) concurrent collector is not active and (2) the butterfly does not contain any property storage. // This is because during deallocation concurrent collector can access butterfly and DFG concurrent compilers accesses properties. // Objects with no properties are common in arrays, and we are focusing on very large array crafted by repeating Array#push, so... that's fine! - bool canRealloc = !propertyCapacity && !vm.heap.mutatorShouldBeFenced() && std::bit_cast(theBase)->isPreciseAllocation(); + HeapCell* baseCell = std::bit_cast(theBase); + // A snapshot (immortal) allocation is left exactly as it is: growing it takes the allocate-and-copy path below instead. + bool canRealloc = !propertyCapacity && !vm.heap.mutatorShouldBeFenced() && baseCell->isPreciseAllocation() && !baseCell->preciseAllocation().isImmortal(); if (canRealloc) { void* newBase = vm.auxiliarySpace().reallocatePreciseAllocationNonVirtual(vm, std::bit_cast(theBase), newSize, &deferralContext, AllocationFailureMode::ReturnNull); if (!newBase) diff --git a/Source/JavaScriptCore/runtime/CachePayload.h b/Source/JavaScriptCore/runtime/CachePayload.h index 634350ac68e0..d2293947a87c 100644 --- a/Source/JavaScriptCore/runtime/CachePayload.h +++ b/Source/JavaScriptCore/runtime/CachePayload.h @@ -38,6 +38,9 @@ class CachePayload { JS_EXPORT_PRIVATE static CachePayload makeMallocPayload(MallocSpan&&); JS_EXPORT_PRIVATE static CachePayload makePayloadWithDestructor(std::span, Destructor&&); JS_EXPORT_PRIVATE static CachePayload makeEmptyPayload(); + // Only an explicit embedder promise makes a payload borrowable; nothing is inferred from how it is stored. + bool isPersistent() const { return m_isPersistent; } + void setIsPersistent() { m_isPersistent = true; } // embedder promises the bytes outlive every VM use (e.g. a section of the running executable) JS_EXPORT_PRIVATE CachePayload(CachePayload&&); JS_EXPORT_PRIVATE ~CachePayload(); @@ -50,6 +53,7 @@ class CachePayload { explicit CachePayload(DataType&&, Destructor&& = {nullptr}); DataType m_data; + bool m_isPersistent { false }; Destructor m_destructor { nullptr }; }; diff --git a/Source/JavaScriptCore/runtime/CachedBytecode.h b/Source/JavaScriptCore/runtime/CachedBytecode.h index 99669b1dfba5..e8564f77c67c 100644 --- a/Source/JavaScriptCore/runtime/CachedBytecode.h +++ b/Source/JavaScriptCore/runtime/CachedBytecode.h @@ -71,6 +71,8 @@ class CachedBytecode : public RefCounted { std::span span() const LIFETIME_BOUND { return m_payload.span(); } size_t size() const { return m_payload.size(); } + bool payloadIsPersistent() const { return m_payload.isPersistent(); } + void setPayloadIsPersistent() { m_payload.setIsPersistent(); } bool hasUpdates() const { return !m_updates.isEmpty(); } size_t sizeForUpdate() const { return m_size; } diff --git a/Source/JavaScriptCore/runtime/CachedTypes.cpp b/Source/JavaScriptCore/runtime/CachedTypes.cpp index 261fc8b0e5e8..3b15f31b778d 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.cpp +++ b/Source/JavaScriptCore/runtime/CachedTypes.cpp @@ -25,6 +25,7 @@ #include "config.h" #include "CachedTypes.h" +#include #include "BaselineJITCode.h" #include "BuiltinNames.h" @@ -56,6 +57,11 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { +bool Decoder::canBorrowPayload() const +{ + return Options::useBorrowedBytecodeFromCache() && m_cachedBytecode->payloadIsPersistent(); +} + namespace Yarr { enum class Flags : uint16_t; } @@ -293,8 +299,10 @@ Decoder::Decoder(VM& vm, Ref cachedBytecode, RefPtr Decoder::create(VM& vm, Ref cachedBytecode, RefPtr provider) @@ -349,13 +357,14 @@ void Decoder::setHandleForTDZEnvironment(CompactTDZEnvironment* environment, con void Decoder::addLeafExecutable(const UnlinkedFunctionExecutable* executable, ptrdiff_t offset) { + if (!m_recordsLeafExecutables) + return; m_cachedBytecode->leafExecutables().add(executable, offset); } -template -void Decoder::addFinalizer(const Functor& fn) +void Decoder::addFinalizer(void* ptr, void (*finalizer)(void*)) { - m_finalizers.append(fn); + m_finalizers.append({ ptr, finalizer }); } RefPtr Decoder::provider() const @@ -487,6 +496,11 @@ class CachedArray : public VariableLengthObject { for (unsigned i = 0; i < size; ++i) ::JSC::decode(decoder, buffer[i], array[i], args...); } + const T* borrow() const + { + static_assert(std::is_trivially_copyable_v, "borrowing is a raw view of the encoded bytes"); + return this->isEmpty() ? nullptr : this->template buffer(); + } }; template> @@ -512,8 +526,11 @@ class CachedPtr : public VariableLengthObject { encoder.cachePtr(src, encoder.offsetOf(cachedObject)); } + // Only for objects whose lifetime the Decoder itself guarantees (CachedRefPtr adds a finalizer holding a ref; + // TDZ environments are owned by the VM map or a finalizer). Uniquely-owned objects and GC cells can die while + // the Decoder lives, so they must always be decoded fresh via decode(). template - Source* decode(Decoder& decoder, bool& isNewAllocation, Args&&... args) const + Source* decodeShared(Decoder& decoder, bool& isNewAllocation, Args&&... args) const { if (this->isEmpty()) { isNewAllocation = false; @@ -535,8 +552,9 @@ class CachedPtr : public VariableLengthObject { template Source* decode(Decoder& decoder, Args&&... args) const { - bool unusedIsNewAllocation; - return decode(decoder, unusedIsNewAllocation, std::forward(args)...); + if (this->isEmpty()) + return nullptr; + return get()->decode(decoder, std::forward(args)...); } const T* NODELETE operator->() const { return get(); } @@ -570,12 +588,12 @@ class CachedRefPtr : public CachedObject> { RefPtr decode(Decoder& decoder) const { bool isNewAllocation; - Source* decodedPtr = m_ptr.decode(decoder, isNewAllocation); + Source* decodedPtr = m_ptr.decodeShared(decoder, isNewAllocation); if (!decodedPtr) return nullptr; if (isNewAllocation) { - decoder.addFinalizer([=] { - WTF::DefaultRefDerefTraits::derefIfNotNull(decodedPtr); + decoder.addFinalizer(decodedPtr, [](void* ptr) { + WTF::DefaultRefDerefTraits::derefIfNotNull(static_cast(ptr)); }); } auto result = adoptRef(decodedPtr); @@ -634,6 +652,14 @@ class CachedVector : public VariableLengthObject, InlineCap ::JSC::encode(encoder, buffer[i], vector[i]); } + std::span borrow() const // raw view of the encoded elements (only meaningful for trivially-encoded T like uint8_t) + { + static_assert(std::is_trivially_copyable_v, "borrowing is a raw view of the encoded bytes"); + if (!m_size) + return { }; + return { this->template buffer(), m_size }; + } + template void decode(Decoder& decoder, VectorContainer& vector, Args... args) const { @@ -1066,6 +1092,8 @@ class CachedExpressionInfo : public CachedObject { std::unique_ptr decode(Decoder& decoder) const { + if (decoder.canBorrowPayload() && !m_storage.isEmpty()) + return ExpressionInfo::createBorrowed(m_numberOfChapters, m_numberOfEncodedInfo, m_numberOfEncodedInfoExtensions, m_storage.borrow()); auto info = ExpressionInfo::createUninitialized(m_numberOfChapters, m_numberOfEncodedInfo, m_numberOfEncodedInfoExtensions); m_storage.decode(decoder, info->payload(), info->payloadSize()); return info; @@ -1172,7 +1200,7 @@ class CachedCompactTDZEnvironmentMapHandle : public CachedObjectget(environment, isNewEntry); if (!isNewEntry) { - decoder.addFinalizer([=] { - delete environment; + decoder.addFinalizer(environment, [](void* ptr) { + delete static_cast(ptr); }); } decoder.setHandleForTDZEnvironment(environment, handle); @@ -1514,11 +1542,16 @@ class CachedInstructionStream : public CachedObject { public: void encode(Encoder& encoder, const JSInstructionStream& stream) { + RELEASE_ASSERT(!stream.isBorrowed()); // a borrowed stream's bytes live in the payload being read, not in m_instructions m_instructions.encode(encoder, stream.m_instructions); } JSInstructionStream* decode(Decoder& decoder) const { + if (decoder.canBorrowPayload()) { + // The cache outlives the VM's use of it (mmap'd / embedded): point at the bytes instead of copying them. + return new JSInstructionStream(m_instructions.borrow(), JSInstructionStream::Borrow); + } Vector instructionsVector; m_instructions.decode(decoder, instructionsVector); return new JSInstructionStream(WTF::move(instructionsVector)); @@ -2484,9 +2517,10 @@ ALWAYS_INLINE UnlinkedFunctionExecutable::UnlinkedFunctionExecutable(Decoder& de if (!cachedExecutable.unlinkedCodeBlockForCall().isEmpty() || !cachedExecutable.unlinkedCodeBlockForConstruct().isEmpty()) { checkBounds(m_cachedCodeBlockForCallOffset, cachedExecutable.unlinkedCodeBlockForCall()); checkBounds(m_cachedCodeBlockForConstructOffset, cachedExecutable.unlinkedCodeBlockForConstruct()); - if (m_isCached) + if (m_isCached) { m_decoder = &decoder; - else + m_cachedRecordOffset = static_cast(decoder.offsetOf(&cachedExecutable)); + } else m_decoder = nullptr; } @@ -2761,7 +2795,7 @@ UnlinkedCodeBlock* decodeCodeBlockImpl(VM& vm, const SourceCodeKey& key, Ref(cachedBytecode->span().data()); - Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode), &key.source().provider()); + Ref decoder = vm.ensureBytecodeCacheDecoder(WTF::move(cachedBytecode), &key.source().provider()); std::pair entry; { DeferGC deferGC(vm); @@ -2796,6 +2830,30 @@ void decodeFunctionCodeBlock(Decoder& decoder, int32_t cachedFunctionCodeBlockOf cachedCodeBlock->decode(decoder, codeBlock, owner); } +void decodeFunctionCodeBlockForReDecode(Decoder& decoder, int32_t cachedFunctionCodeBlockOffset, WriteBarrier& codeBlock, const JSCell* owner) +{ + SetForScope reDecoding(decoder.m_recordsLeafExecutables, false); + decodeFunctionCodeBlock(decoder, cachedFunctionCodeBlockOffset, codeBlock, owner); +} + +void decodeFunctionCodeBlockFromExecutableRecord(Decoder& decoder, int32_t cachedFunctionExecutableOffset, CodeSpecializationKind kind, WriteBarrier& codeBlock, const UnlinkedFunctionExecutable& executable) +{ + ASSERT(decoder.vm().heap.isDeferred()); + if (cachedFunctionExecutableOffset <= 0 || static_cast(cachedFunctionExecutableOffset) + sizeof(CachedFunctionExecutable) > decoder.size()) + return; + auto* cachedExecutable = static_cast(decoder.ptrForOffsetFromBase(cachedFunctionExecutableOffset)); + // The offset was recorded against the payload this executable was first decoded from; the provider's payload could have been + // replaced or rewritten since, so the record must describe this very function before anything in it is trusted. + if (cachedExecutable->startOffset() != executable.startOffset() || cachedExecutable->unlinkedFunctionStart() != executable.unlinkedFunctionStart() + || cachedExecutable->lineCount() != executable.lineCount() || cachedExecutable->parameterCount() != executable.parameterCount()) + return; + const auto& cachedCodeBlock = kind == CodeSpecializationKind::CodeForCall ? cachedExecutable->unlinkedCodeBlockForCall() : cachedExecutable->unlinkedCodeBlockForConstruct(); + if (cachedCodeBlock.isEmpty()) + return; + SetForScope reDecoding(decoder.m_recordsLeafExecutables, false); + cachedCodeBlock.decode(decoder, codeBlock, &executable); +} + } // namespace JSC WTF_ALLOW_UNSAFE_BUFFER_USAGE_END diff --git a/Source/JavaScriptCore/runtime/CachedTypes.h b/Source/JavaScriptCore/runtime/CachedTypes.h index 3ce6592e5b92..6465292056e1 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.h +++ b/Source/JavaScriptCore/runtime/CachedTypes.h @@ -82,6 +82,7 @@ class Decoder : public RefCounted { public: static Ref create(VM&, Ref, RefPtr = nullptr); + bool canBorrowPayload() const; // the embedder promised the payload outlives every use, so decoded objects may alias it ~Decoder(); @@ -96,19 +97,24 @@ class Decoder : public RefCounted { void setHandleForTDZEnvironment(CompactTDZEnvironment*, const CompactTDZEnvironmentMap::Handle&); void addLeafExecutable(const UnlinkedFunctionExecutable*, ptrdiff_t); RefPtr NODELETE provider() const; + CachedBytecode& cachedBytecode() const { return m_cachedBytecode.get(); } - template - void addFinalizer(const Functor&); + void addFinalizer(void*, void (*)(void*)); + void setIsRegisteredWithVM() { m_isRegisteredWithVM = true; } private: + friend void decodeFunctionCodeBlockForReDecode(Decoder&, int32_t, WriteBarrier&, const JSCell*); + friend void decodeFunctionCodeBlockFromExecutableRecord(Decoder&, int32_t, CodeSpecializationKind, WriteBarrier&, const UnlinkedFunctionExecutable&); Decoder(VM&, Ref, RefPtr); VM& m_vm; const Ref m_cachedBytecode; + bool m_recordsLeafExecutables { true }; // cleared for the duration of a re-decode: nothing decoded then is ever written back UncheckedKeyHashMap m_offsetToPtrMap; - Vector> m_finalizers; + Vector> m_finalizers; UncheckedKeyHashMap m_environmentToHandleMap; RefPtr m_provider; + bool m_isRegisteredWithVM { false }; }; JS_EXPORT_PRIVATE RefPtr encodeCodeBlock(VM&, const SourceCodeKey&, const UnlinkedCodeBlock*); @@ -127,6 +133,8 @@ std::optional decodeSourceCodeKey(VM& vm, Ref cac JS_EXPORT_PRIVATE RefPtr encodeFunctionCodeBlock(VM&, const UnlinkedFunctionCodeBlock*, BytecodeCacheError&); JS_EXPORT_PRIVATE void decodeFunctionCodeBlock(Decoder&, int32_t cachedFunctionCodeBlockOffset, WriteBarrier&, const JSCell*); +JS_EXPORT_PRIVATE void decodeFunctionCodeBlockForReDecode(Decoder&, int32_t cachedFunctionCodeBlockOffset, WriteBarrier&, const JSCell* owner); // as decodeFunctionCodeBlock, for code that is being decoded again: records nothing +JS_EXPORT_PRIVATE void decodeFunctionCodeBlockFromExecutableRecord(Decoder&, int32_t cachedFunctionExecutableOffset, CodeSpecializationKind, WriteBarrier&, const UnlinkedFunctionExecutable&); bool isCachedBytecodeStillValid(VM&, Ref, const SourceCodeKey&, SourceCodeType); diff --git a/Source/JavaScriptCore/runtime/IntlCollator.cpp b/Source/JavaScriptCore/runtime/IntlCollator.cpp index 6d745aa0e534..405f8d6184e9 100644 --- a/Source/JavaScriptCore/runtime/IntlCollator.cpp +++ b/Source/JavaScriptCore/runtime/IntlCollator.cpp @@ -28,6 +28,7 @@ #include "config.h" #include "IntlCollator.h" +#include "TopExceptionScope.h" #include "IntlObjectInlines.h" #include "JSBoundFunction.h" #include "JSCInlines.h" @@ -224,8 +225,21 @@ void IntlCollator::initializeCollator(JSGlobalObject* globalObject, JSValue loca } dataLogLnIf(IntlCollatorInternal::verbose, "locale:(", resolved.locale, "),dataLocaleWithExtensions:(", dataLocaleWithExtensions, ")"); + m_icuLocale = dataLocaleWithExtensions; +#if USE(BUN_JSC_ADDITIONS) + m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); +#endif + scope.release(); // whatever the opener throws is this initialization's to propagate + openCollator(globalObject, ignorePunctuation); +} + +// Opens and configures the ICU collator from the resolved fields. Called once when the object is initialized and again in a +// process restored from a snapshot, whose ICU is not the one the existing handle came from. +void IntlCollator::openCollator(JSGlobalObject* globalObject, TriState ignorePunctuation) +{ + auto scope = DECLARE_THROW_SCOPE(globalObject->vm()); UErrorCode status = U_ZERO_ERROR; - m_collator = std::unique_ptr(ucol_open(dataLocaleWithExtensions.data(), &status)); + m_collator = std::unique_ptr(ucol_open(m_icuLocale.data(), &status)); if (U_FAILURE(status)) { throwTypeError(globalObject, scope, "failed to initialize Collator"_s); return; @@ -281,10 +295,34 @@ void IntlCollator::initializeCollator(JSGlobalObject* globalObject, JSValue loca } } +UCollator* IntlCollator::collatorForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_startupSnapshotEpoch != vm.startupSnapshotEpoch()) [[unlikely]] { + (void)m_collator.release(); // the handle belongs to the process that built the snapshot; its ICU is gone + m_canDoASCIIUCADUCETComparison = TriState::Indeterminate; // a verdict about the old ICU's rules; this machine's may differ + { + auto catchScope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + const_cast(this)->openCollator(globalObject, m_ignorePunctuation ? TriState::True : TriState::False); + if (catchScope.exception()) [[unlikely]] { + (void)catchScope.tryClearException(); + RELEASE_ASSERT_WITH_MESSAGE(false, "Intl.Collator could not reopen its ICU collator in the restored process (different ICU?)"); + } + } + m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); + } +#else + UNUSED_PARAM(globalObject); +#endif + return m_collator.get(); +} + // https://tc39.es/ecma402/#sec-collator-comparestrings UCollationResult IntlCollator::compareStrings(JSGlobalObject* globalObject, StringView x, StringView y) const { - ASSERT(m_collator); + UCollator* collator = collatorForThisProcess(globalObject); // before the ASCII probe below, which uses the handle + ASSERT(collator); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -304,14 +342,14 @@ UCollationResult IntlCollator::compareStrings(JSGlobalObject* globalObject, Stri if (x.is8Bit() && y.is8Bit() && x.containsOnlyASCII() && y.containsOnlyASCII()) { auto xCharacters = byteCast(x.span8()); auto yCharacters = byteCast(y.span8()); - return ucol_strcollUTF8(m_collator.get(), xCharacters.data(), xCharacters.size(), yCharacters.data(), yCharacters.size(), &status); + return ucol_strcollUTF8(collator, xCharacters.data(), xCharacters.size(), yCharacters.data(), yCharacters.size(), &status); } return std::nullopt; }()); if (!result) - result = ucol_strcoll(m_collator.get(), x.upconvertedCharacters(), x.length(), y.upconvertedCharacters(), y.length()); + result = ucol_strcoll(collator, x.upconvertedCharacters(), x.length(), y.upconvertedCharacters(), y.length()); if (U_FAILURE(status)) { throwException(globalObject, scope, createError(globalObject, "Failed to compare strings."_s)); diff --git a/Source/JavaScriptCore/runtime/IntlCollator.h b/Source/JavaScriptCore/runtime/IntlCollator.h index a6f93ce15d99..519882b72b77 100644 --- a/Source/JavaScriptCore/runtime/IntlCollator.h +++ b/Source/JavaScriptCore/runtime/IntlCollator.h @@ -61,6 +61,8 @@ class IntlCollator final : public JSNonFinalObject { DECLARE_VISIT_CHILDREN; void initializeCollator(JSGlobalObject*, JSValue locales, JSValue optionsValue); + void openCollator(JSGlobalObject*, TriState ignorePunctuation); + UCollator* collatorForThisProcess(JSGlobalObject*) const; UCollationResult compareStrings(JSGlobalObject*, StringView, StringView) const; JSObject* resolvedOptions(JSGlobalObject*) const; @@ -100,7 +102,11 @@ class IntlCollator final : public JSNonFinalObject { static ASCIILiteral caseFirstString(CaseFirst); WriteBarrier m_boundCompare; - std::unique_ptr m_collator; + mutable std::unique_ptr m_collator; + CString m_icuLocale; +#if USE(BUN_JSC_ADDITIONS) + mutable unsigned m_startupSnapshotEpoch { 0 }; +#endif String m_locale; String m_collation; diff --git a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp index 48fe49442564..d05d32573190 100644 --- a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp +++ b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.cpp @@ -1088,6 +1088,11 @@ void IntlDateTimeFormat::initializeDateTimeFormat(JSGlobalObject* globalObject, UErrorCode status = U_ZERO_ERROR; String timeZoneForICU = impl->m_timeZone.toICUString(); impl->m_dateFormat = openDateFormat(dataLocaleWithExtensions, timeZoneForICU, patternBuffer.span(), status); +#if USE(BUN_JSC_ADDITIONS) + impl->m_icuPattern = patternBuffer; + impl->m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); + m_intervalFormatEpoch = impl->m_startupSnapshotEpoch; +#endif if (U_FAILURE(status)) [[unlikely]] { throwTypeError(globalObject, scope, "failed to initialize DateTimeFormat"_s); return; @@ -1394,8 +1399,34 @@ static void NODELETE replaceNarrowNoBreakSpaceOrThinSpaceWithNormalSpace(Contain } // https://tc39.es/proposal-temporal/#sec-formatdatetime + +void IntlDateTimeFormat::ensureICUObjectsForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_intervalFormatEpoch != vm.startupSnapshotEpoch()) [[unlikely]] { + (void)m_dateIntervalFormat.release(); // rebuilt on demand by formatRange + m_intervalFormatEpoch = vm.startupSnapshotEpoch(); + } + auto* impl = const_cast(m_impl.get()); + if (!impl || impl->m_startupSnapshotEpoch == vm.startupSnapshotEpoch()) [[likely]] + return; + // Every handle here was opened by the process that built the snapshot, against an ICU this process does not share (on macOS + // it is the system's). The main formatter is opened again from what it was opened with; the ones built lazily are dropped + // and come back on demand. The old handles are leaked on purpose: closing them would hand foreign memory to this ICU. + (void)impl->m_dateFormat.release(); + (void)impl->m_temporalFormatterCache.release(); // its formatters come back on demand + UErrorCode status = U_ZERO_ERROR; + impl->m_dateFormat = openDateFormat(impl->m_dataLocaleWithExtensions, impl->m_timeZone.toICUString(), impl->m_icuPattern.span(), status); + if (U_SUCCESS(status)) + impl->m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); +#else + UNUSED_PARAM(globalObject); +#endif +} JSValue IntlDateTimeFormat::format(JSGlobalObject* globalObject, double value) const { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); @@ -1988,6 +2019,7 @@ static JSValue buildFormattedDateIntervalParts(JSGlobalObject* globalObject, con // https://tc39.es/proposal-temporal/#sec-formatdatetimerangetoparts JSValue IntlDateTimeFormat::formatRangeToParts(JSGlobalObject* globalObject, JSValue xValue, JSValue yValue) { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -2384,6 +2416,7 @@ std::unique_ptr IntlDateTim // FormatDateTime(dateTimeFormat, x) — dispatches through HandleDateTimeValue for Temporal objects. JSValue IntlDateTimeFormat::format(JSGlobalObject* globalObject, JSValue x) const { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -2414,6 +2447,7 @@ JSValue IntlDateTimeFormat::format(JSGlobalObject* globalObject, JSValue x) cons // https://tc39.es/proposal-temporal/#sec-formatdatetimetoparts JSValue IntlDateTimeFormat::formatToParts(JSGlobalObject* globalObject, JSValue x) const { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -2467,6 +2501,7 @@ IntlDateTimeFormat::createTemporalIntervalFormat(UDateFormat* tempFormat, Tempor // https://tc39.es/proposal-temporal/#sec-formatdatetimerange JSValue IntlDateTimeFormat::formatRange(JSGlobalObject* globalObject, JSValue xValue, JSValue yValue) { + ensureICUObjectsForThisProcess(globalObject); ASSERT(m_impl->m_dateFormat); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h index 6805a22d0ff6..6674a178ddde 100644 --- a/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h +++ b/Source/JavaScriptCore/runtime/IntlDateTimeFormat.h @@ -121,6 +121,7 @@ class IntlDateTimeFormat final : public JSNonFinalObject { JSValue formatRange(JSGlobalObject*, JSValue startDate, JSValue endDate); JSValue formatRangeToParts(JSGlobalObject*, JSValue startDate, JSValue endDate); JSObject* resolvedOptions(JSGlobalObject*) const; + void ensureICUObjectsForThisProcess(JSGlobalObject*) const; // snapshot restore: the ICU handles came from another process static bool isTemporalObject(JSValue); static bool sameTemporalType(JSValue, JSValue); @@ -213,7 +214,10 @@ class IntlDateTimeFormat final : public JSNonFinalObject { static String buildSkeleton(Weekday, Era, Year, Month, Day, TriState, HourCycle, Hour, DayPeriod, Minute, Second, unsigned, TimeZoneName); WriteBarrier m_boundFormat; - std::unique_ptr m_dateIntervalFormat; + mutable std::unique_ptr m_dateIntervalFormat; +#if USE(BUN_JSC_ADDITIONS) + mutable unsigned m_intervalFormatEpoch { 0 }; // this object's own lazily-built handle; the shared impl has its own epoch +#endif RefPtr m_impl; }; @@ -258,6 +262,10 @@ class IntlDateTimeFormatImpl : public RefCounted { IntlDateTimeFormat::DateTimeStyle m_dateStyle { IntlDateTimeFormat::DateTimeStyle::None }; IntlDateTimeFormat::DateTimeStyle m_timeStyle { IntlDateTimeFormat::DateTimeStyle::None }; bool m_anyPresent { false }; +#if USE(BUN_JSC_ADDITIONS) + Vector m_icuPattern; // the pattern m_dateFormat was opened with, so it can be opened again after a snapshot restore + unsigned m_startupSnapshotEpoch { 0 }; +#endif Vector m_userSkeleton; // user's explicit options as skeleton, before defaults injection; used by computeGetDateTimeFormat std::unique_ptr m_dateFormat; mutable std::unique_ptr m_temporalFormatterCache; diff --git a/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp b/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp index ce3def9f190e..ff23773ff351 100644 --- a/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp +++ b/Source/JavaScriptCore/runtime/IntlDisplayNames.cpp @@ -26,6 +26,7 @@ #include "config.h" #include "IntlDisplayNames.h" +#include "TopExceptionScope.h" #include "IntlCache.h" #include "IntlObjectInlines.h" #include "JSCInlines.h" @@ -105,6 +106,18 @@ void IntlDisplayNames::initializeDisplayNames(JSGlobalObject* globalObject, JSVa m_languageDisplay = intlOption(globalObject, options, vm.propertyNames->languageDisplay, { { "dialect"_s, LanguageDisplay::Dialect }, { "standard"_s, LanguageDisplay::Standard } }, "languageDisplay must be either \"dialect\" or \"standard\""_s, LanguageDisplay::Dialect); RETURN_IF_EXCEPTION(scope, void()); +#if USE(BUN_JSC_ADDITIONS) + m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); +#endif + scope.release(); // whatever the opener throws is this initialization's to propagate + openICUObjects(globalObject); +} + +// Builds the ICU objects from the resolved fields: once at initialization, and again in a process restored from a +// snapshot, whose ICU is not the one the existing handles came from. +void IntlDisplayNames::openICUObjects(JSGlobalObject* globalObject) +{ + auto scope = DECLARE_THROW_SCOPE(globalObject->vm()); UErrorCode status = U_ZERO_ERROR; UDisplayContext contexts[] = { @@ -134,9 +147,34 @@ void IntlDisplayNames::initializeDisplayNames(JSGlobalObject* globalObject, JSVa } } +void IntlDisplayNames::ensureICUObjectsForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_startupSnapshotEpoch == vm.startupSnapshotEpoch()) [[likely]] + return; + auto* self = const_cast(this); + // Opened by the process that built the snapshot; released, never closed, since this ICU never allocated them. + (void)self->m_displayNames.release(); + { + auto catchScope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + self->openICUObjects(globalObject); + if (catchScope.exception()) [[unlikely]] { + (void)catchScope.tryClearException(); + RELEASE_ASSERT_WITH_MESSAGE(false, "%s could not reopen its ICU objects in the restored process (different ICU?)", "IntlDisplayNames"); + } + } + if (self->m_displayNames) + self->m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); +#else + UNUSED_PARAM(globalObject); +#endif +} + // https://tc39.es/proposal-intl-displaynames/#sec-Intl.DisplayNames.prototype.of JSValue IntlDisplayNames::of(JSGlobalObject* globalObject, JSValue codeValue) const { + ensureICUObjectsForThisProcess(globalObject); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/runtime/IntlDisplayNames.h b/Source/JavaScriptCore/runtime/IntlDisplayNames.h index 17f257cb04a5..0c23a47a4514 100644 --- a/Source/JavaScriptCore/runtime/IntlDisplayNames.h +++ b/Source/JavaScriptCore/runtime/IntlDisplayNames.h @@ -55,6 +55,10 @@ class IntlDisplayNames final : public JSNonFinalObject { DECLARE_INFO; + + void openICUObjects(JSGlobalObject*); + + void ensureICUObjectsForThisProcess(JSGlobalObject*) const; void initializeDisplayNames(JSGlobalObject*, JSValue localesValue, JSValue optionsValue); JSValue of(JSGlobalObject*, JSValue) const; @@ -76,6 +80,9 @@ class IntlDisplayNames final : public JSNonFinalObject { using ULocaleDisplayNamesDeleter = ICUDeleter; std::unique_ptr m_displayNames; +#if USE(BUN_JSC_ADDITIONS) + unsigned m_startupSnapshotEpoch { 0 }; +#endif String m_locale; // FIXME: We should store it only when m_type is Currency. // https://bugs.webkit.org/show_bug.cgi?id=213773 diff --git a/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp b/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp index 8de3d534b903..018e2d4f08ce 100644 --- a/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp +++ b/Source/JavaScriptCore/runtime/IntlDurationFormat.cpp @@ -194,6 +194,24 @@ static PropertyName NODELETE displayName(VM& vm, TemporalUnit unit) } // https://tc39.es/proposal-intl-duration-format/#sec-Intl.DurationFormat +static UListFormatterWidth toUListFormatterWidth(IntlDurationFormat::Style style) +{ + // 6. Let listStyle be durationFormat.[[Style]]. + // 7. If listStyle is "digital", then + // a. Set listStyle to "short". + // 8. Perform ! CreateDataPropertyOrThrow(lfOpts, "style", listStyle). + switch (style) { + case IntlDurationFormat::Style::Long: + return ULISTFMT_WIDTH_WIDE; + case IntlDurationFormat::Style::Short: + case IntlDurationFormat::Style::Digital: + return ULISTFMT_WIDTH_SHORT; + case IntlDurationFormat::Style::Narrow: + return ULISTFMT_WIDTH_NARROW; + } + return ULISTFMT_WIDTH_WIDE; +} + void IntlDurationFormat::initializeDurationFormat(JSGlobalObject* globalObject, JSValue locales, JSValue optionsValue) { VM& vm = globalObject->vm(); @@ -264,22 +282,6 @@ void IntlDurationFormat::initializeDurationFormat(JSGlobalObject* globalObject, RETURN_IF_EXCEPTION(scope, void()); { - auto toUListFormatterWidth = [](Style style) { - // 6. Let listStyle be durationFormat.[[Style]]. - // 7. If listStyle is "digital", then - // a. Set listStyle to "short". - // 8. Perform ! CreateDataPropertyOrThrow(lfOpts, "style", listStyle). - switch (style) { - case Style::Long: - return ULISTFMT_WIDTH_WIDE; - case Style::Short: - case Style::Digital: - return ULISTFMT_WIDTH_SHORT; - case Style::Narrow: - return ULISTFMT_WIDTH_NARROW; - } - return ULISTFMT_WIDTH_WIDE; - }; // 5. Perform ! CreateDataPropertyOrThrow(lfOpts, "type", "unit"). UErrorCode status = U_ZERO_ERROR; @@ -289,6 +291,29 @@ void IntlDurationFormat::initializeDurationFormat(JSGlobalObject* globalObject, return; } } +#if USE(BUN_JSC_ADDITIONS) + m_startupSnapshotEpoch = globalObject->vm().startupSnapshotEpoch(); +#endif +} + +void IntlDurationFormat::ensureICUObjectsForThisProcess(JSGlobalObject* globalObject) const +{ +#if USE(BUN_JSC_ADDITIONS) + VM& vm = globalObject->vm(); + if (m_startupSnapshotEpoch == vm.startupSnapshotEpoch()) [[likely]] + return; + auto* self = const_cast(this); + // Opened by the process that built the snapshot, against an ICU this process does not share: released, never closed. The + // per-unit number formatters are built lazily, so dropping the cache is enough for them; the list formatter is opened again. + (void)self->m_formatterCache.release(); + (void)self->m_listFormat.release(); + UErrorCode status = U_ZERO_ERROR; + self->m_listFormat = std::unique_ptr(ulistfmt_openForType(m_locale.utf8().data(), ULISTFMT_TYPE_UNITS, toUListFormatterWidth(m_style), &status)); + if (U_SUCCESS(status)) + self->m_startupSnapshotEpoch = vm.startupSnapshotEpoch(); +#else + UNUSED_PARAM(globalObject); +#endif } const String& IntlDurationFormat::numberingSystem() const @@ -690,6 +715,7 @@ UNumberFormatter* IntlDurationFormat::createNumberFormatterIfNecessary(JSGlobalO // https://tc39.es/proposal-intl-duration-format/#sec-Intl.DurationFormat.prototype.format JSValue IntlDurationFormat::format(JSGlobalObject* globalObject, ISO8601::Duration duration) const { + ensureICUObjectsForThisProcess(globalObject); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); @@ -733,6 +759,7 @@ JSValue IntlDurationFormat::format(JSGlobalObject* globalObject, ISO8601::Durati // https://tc39.es/proposal-intl-duration-format/#sec-Intl.DurationFormat.prototype.formatToParts JSValue IntlDurationFormat::formatToParts(JSGlobalObject* globalObject, ISO8601::Duration duration) const { + ensureICUObjectsForThisProcess(globalObject); VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/runtime/IntlDurationFormat.h b/Source/JavaScriptCore/runtime/IntlDurationFormat.h index 45b656c8cad1..ee297d51f545 100644 --- a/Source/JavaScriptCore/runtime/IntlDurationFormat.h +++ b/Source/JavaScriptCore/runtime/IntlDurationFormat.h @@ -59,6 +59,8 @@ class IntlDurationFormat final : public JSNonFinalObject { DECLARE_VISIT_CHILDREN; + + void ensureICUObjectsForThisProcess(JSGlobalObject*) const; void initializeDurationFormat(JSGlobalObject*, JSValue localesValue, JSValue optionsValue); JSValue format(JSGlobalObject*, ISO8601::Duration) const; @@ -112,6 +114,10 @@ class IntlDurationFormat final : public JSNonFinalObject { }; std::unique_ptr m_listFormat; +#if USE(BUN_JSC_ADDITIONS) + + unsigned m_startupSnapshotEpoch { 0 }; +#endif mutable std::unique_ptr m_formatterCache; String m_locale; String m_dataLocale; diff --git a/Source/JavaScriptCore/runtime/IntlListFormat.cpp b/Source/JavaScriptCore/runtime/IntlListFormat.cpp index 4f8221b84cfa..65a456536c9b 100644 --- a/Source/JavaScriptCore/runtime/IntlListFormat.cpp +++ b/Source/JavaScriptCore/runtime/IntlListFormat.cpp @@ -27,6 +27,7 @@ #include "config.h" #include "IntlListFormat.h" +#include "TopExceptionScope.h" #include "IntlObjectInlines.h" #include "IntlPartObject.h" #include "IteratorOperations.h" @@ -109,6 +110,18 @@ void IntlListFormat::initializeListFormat(JSGlobalObject* globalObject, JSValue m_style = intlOption