From c2fd8368c3c214e8f7f1f4be34b3c83db0ec0404 Mon Sep 17 00:00:00 2001 From: Sosuke Suzuki Date: Fri, 28 Aug 2026 00:56:21 +0900 Subject: [PATCH] Save and restore generator locals in bulk with dedicated bytecodes Generatorification spills every live local with its own op_put_to_scope at every suspension point and reloads each with op_get_from_scope on resume, so generator and async-function bytecode grows as live locals x suspension points. Large async generators compile to bytecode that is mostly this save/restore code, plus Baseline machine code and linked metadata per local per suspension point. Replace the per-local sequences with two bytecodes, op_save_generator_locals and op_restore_generator_locals, which copy all live locals at once. Each op names a per-suspension-point liveness bit vector and the function-wide union of live locals (both stored in the existing UnlinkedCodeBlock rare-data bit vector table, deduplicated by content); a local's frame slot is its rank within the union, so slots stay stable across suspension points and the generator frame is sized by the number of ever-live locals. Locals no longer need per-local identifiers, SymbolTableEntries, watchpoint sets, or metadata. LLInt runs one C++ slow path per op with a single write barrier; the Baseline JIT emits an unrolled load/store sequence (restore also stores into the op's run of consecutive value profiles); the DFG parser expands the ops into the same PutClosureVar / GetClosureVar nodes as before, so optimized code is unchanged, with OSR-exit value feedback routed through lazy operand profiles keyed by the restored local. The new encoding is behind Options::useGeneratorBulkSaveRestore (default true). With the option off, generatorification emits the previous per-local op_put_to_scope / op_get_from_scope sequences. Every tier executes both encodings, so cached bytecode produced under either setting runs regardless of the current value. --- .../stress/generator-save-restore-locals.js | 175 ++++++++++++++++++ .../bytecode/BytecodeGeneratorification.cpp | 101 ++++++++-- .../bytecode/BytecodeGeneratorification.h | 14 +- .../JavaScriptCore/bytecode/BytecodeList.rb | 17 ++ .../bytecode/BytecodeUseDef.cpp | 4 + .../JavaScriptCore/bytecode/BytecodeUseDef.h | 14 ++ .../JavaScriptCore/dfg/DFGByteCodeParser.cpp | 32 ++++ Source/JavaScriptCore/dfg/DFGGraph.cpp | 17 ++ Source/JavaScriptCore/jit/JIT.cpp | 2 + Source/JavaScriptCore/jit/JIT.h | 3 + Source/JavaScriptCore/jit/JITInlines.h | 11 +- Source/JavaScriptCore/jit/JITOpcodes.cpp | 33 ++++ .../llint/LowLevelInterpreter.asm | 2 + Source/JavaScriptCore/lol/LOLJIT.cpp | 2 + .../runtime/CommonSlowPaths.cpp | 29 +++ .../JavaScriptCore/runtime/CommonSlowPaths.h | 2 + Source/JavaScriptCore/runtime/OptionsList.h | 1 + 17 files changed, 441 insertions(+), 18 deletions(-) create mode 100644 JSTests/stress/generator-save-restore-locals.js diff --git a/JSTests/stress/generator-save-restore-locals.js b/JSTests/stress/generator-save-restore-locals.js new file mode 100644 index 000000000000..9d1b2928b2c7 --- /dev/null +++ b/JSTests/stress/generator-save-restore-locals.js @@ -0,0 +1,175 @@ +//@ runDefault +//@ runDefault("--useGeneratorBulkSaveRestore=0") +//@ runDefault("--useJIT=0") +//@ runDefault("--useDFGJIT=0") +//@ runDefault("--useConcurrentJIT=0", "--thresholdForJITAfterWarmUp=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeAfterWarmUp=50") +//@ runBytecodeCache + +function shouldBe(actual, expected) { + if (actual !== expected) + throw new Error(`bad value: ${actual}, expected ${expected}`); +} + +// Mixed-type locals, resumed with next/throw. +function* mixed() { + let a = 1, b = 2.5, c = "s", d = { v: 3 }, e; + try { + a += yield a; + b += yield b; + e = yield c + d.v; + } catch (error) { + return [a, b, c, d.v, e, error]; + } + return [a, b, c, d.v, e]; +} +{ + let iterator = mixed(); + shouldBe(iterator.next().value, 1); + shouldBe(iterator.next(10).value, 2.5); + shouldBe(iterator.next(0.5).value, "s3"); + shouldBe(JSON.stringify(iterator.throw("boom").value), '[11,3,"s",3,null,"boom"]'); +} + +// Live sets of various sizes: 70 needs an out-of-line bit vector, 300 needs wide operands. +function makeWide(count) { + let declarations = []; + let names = []; + for (let i = 0; i < count; ++i) { + declarations.push(`let v${i} = ${i};`); + names.push(`v${i}`); + } + return new Function(` + return function* wide() { + ${declarations.join("\n")} + yield 0; + ${names.map((name) => `${name} += 1;`).join("\n")} + yield 1; + return ${names.join(" + ")}; + }`)(); +} +for (let count of [3, 9, 70, 300]) { + let wide = makeWide(count); + let expected = 0; + for (let i = 0; i < count; ++i) + expected += i + 1; + for (let round = 0; round < 3; ++round) { + let iterator = wide(); + shouldBe(iterator.next().value, 0); + shouldBe(iterator.next().value, 1); + shouldBe(iterator.next().value, expected); + } +} + +// Locals under TDZ hold the empty value across a yield. +function* tdz(flag) { + yield 1; + if (flag) { + let later = 5; + yield later; + } + let after = 7; + yield after; + { + yield 2; + let x = 9; + yield x; + } +} +{ + let iterator = tdz(false); + shouldBe(iterator.next().value, 1); + shouldBe(iterator.next().value, 7); + shouldBe(iterator.next().value, 2); + shouldBe(iterator.next().value, 9); +} + +// Saved locals share the lexical environment with captured variables. +function* captured() { + let counter = 0; + let local = 100; + const bump = () => ++counter; + yield bump(); + local += counter; + yield bump() + local; + return [counter, local]; +} +{ + let iterator = captured(); + shouldBe(iterator.next().value, 1); + shouldBe(iterator.next().value, 103); + shouldBe(JSON.stringify(iterator.next().value), "[2,101]"); +} + +// Nothing live across the yields. +function* empty() { + yield 1; + yield 2; +} +{ + let iterator = empty(); + shouldBe(iterator.next().value, 1); + shouldBe(iterator.next().value, 2); + shouldBe(iterator.next().done, true); +} + +// Async function, async generator, and yield*. +async function asyncFunction(promise) { + let a = 1, b = [1, 2, 3], c = "x"; + a += await promise; + for (const v of b) + c += await v; + return a + c; +} +async function* asyncGenerator() { + let accumulator = 0; + for (let i = 0; i < 3; ++i) { + accumulator += await i; + yield accumulator; + } + yield* empty(); + return accumulator; +} +function* delegating() { + return yield* mixed(); +} +let asyncDone = false; +(async () => { + shouldBe(await asyncFunction(Promise.resolve(10)), "11x123"); + let values = []; + for await (const value of asyncGenerator()) + values.push(value); + shouldBe(JSON.stringify(values), "[0,1,3,1,2]"); + asyncDone = true; +})(); +{ + let iterator = delegating(); + iterator.next(); + iterator.next(1); + iterator.next(1); + shouldBe(JSON.stringify(iterator.next().value), '[2,3.5,"s",3,null]'); +} + +// Tier up with int32 locals, then resume with double and string values. +function* counting(count, start) { + let i = 0, accumulator = start, object = { f: 1 }; + while (i < count) { + accumulator = accumulator + object.f; + yield accumulator; + ++i; + } + return accumulator; +} +function drive(start, count) { + let iterator = counting(count, start); + let last; + for (let result = iterator.next(); !result.done; result = iterator.next()) + last = result.value; + return last; +} +for (let i = 0; i < testLoopCount; ++i) + shouldBe(drive(0, 10), 10); +shouldBe(drive(0.5, 10), 10.5); +shouldBe(drive("s", 3), "s111"); + +drainMicrotasks(); +shouldBe(asyncDone, true); diff --git a/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.cpp b/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.cpp index 620a8c4bf5df..07e6e9261c8c 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.cpp +++ b/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.cpp @@ -38,6 +38,7 @@ #include "StrongInlines.h" #include "UnlinkedCodeBlockGenerator.h" #include "UnlinkedMetadataTableInlines.h" +#include namespace JSC { @@ -166,6 +167,9 @@ class BytecodeGeneratorification { return storage; } + void emitBulkSaveAndRestore(BytecodeRewriter&); + void emitPerLocalSaveAndRestore(BytecodeRewriter&); + BytecodeGenerator& m_bytecodeGenerator; JSInstructionStream::Offset m_enterPoint; std::optional m_generatorFrameData; @@ -204,7 +208,6 @@ void BytecodeGeneratorification::run() { // We calculate the liveness at each merge point. This gives us the information which registers should be saved and resumed conservatively. - VM& vm = m_bytecodeGenerator.vm(); { GeneratorLivenessAnalysis pass(*this); pass.run(m_codeBlock, m_instructions); @@ -231,6 +234,89 @@ void BytecodeGeneratorification::run() }); } + if (Options::useGeneratorBulkSaveRestore()) + emitBulkSaveAndRestore(rewriter); + else + emitPerLocalSaveAndRestore(rewriter); + + if (m_generatorFrameData) { + auto instruction = m_instructions.at(m_generatorFrameData->m_point); + rewriter.replaceBytecodeWithFragment(instruction, [&] (BytecodeRewriter::Fragment& fragment) { + if (!m_generatorFrameSymbolTable->scopeSize()) { + // This will cause us to put jsUndefined() into the generator frame's scope value. + fragment.appendInstruction(m_generatorFrameData->m_dst, m_generatorFrameData->m_initialValue); + } else + fragment.appendInstruction(m_generatorFrameData->m_dst, m_generatorFrameData->m_scope, m_generatorFrameData->m_symbolTable, m_generatorFrameData->m_initialValue); + }); + } + + rewriter.execute(); +} + +void BytecodeGeneratorification::emitBulkSaveAndRestore(BytecodeRewriter& rewriter) +{ + BitVector savedLocals; + for (const YieldData& data : m_yields) { + data.liveness.forEachSetBit([&](size_t index) { + savedLocals.set(index); + }); + } + + unsigned firstScopeOffset = m_generatorFrameSymbolTable->scopeSize(); + for (unsigned i = 0, count = savedLocals.bitCount(); i < count; ++i) { + ScopeOffset scopeOffset = m_generatorFrameSymbolTable->takeNextScopeOffset(NoLockingNecessary); + ASSERT_UNUSED(scopeOffset, scopeOffset.offset() == firstScopeOffset + i); + } + + unsigned numberOfLocalBits = savedLocals.size(); + UncheckedKeyHashMap bitVectorIndices; + auto addBitVectorConstant = [&](const BitVector& bitVector) { + return bitVectorIndices.ensure(bitVector, [&] { + return m_codeBlock->addBitVector(BitVector(bitVector)); + }).iterator->value; + }; + unsigned savedLocalsIndex = savedLocals.isEmpty() ? 0 : addBitVectorConstant(savedLocals); + + for (const YieldData& data : m_yields) { + VirtualRegister scope = virtualRegisterForArgumentIncludingThis(static_cast(JSGenerator::Argument::Frame)); + auto instruction = m_instructions.at(data.point); + + if (data.liveness.isEmpty()) { + rewriter.insertFragmentBefore(instruction, [&] (BytecodeRewriter::Fragment& fragment) { + fragment.appendInstruction(data.argument); + }); + rewriter.replaceBytecodeWithFragment(instruction, [&] (BytecodeRewriter::Fragment&) { }); + continue; + } + + BitVector liveLocals; + liveLocals.ensureSize(numberOfLocalBits); + data.liveness.forEachSetBit([&](size_t index) { + liveLocals.quickSet(index); + }); + unsigned liveLocalsIndex = addBitVectorConstant(liveLocals); + + unsigned firstValueProfile = m_bytecodeGenerator.nextValueProfileIndex(); + for (unsigned i = 1, count = data.liveness.bitCount(); i < count; ++i) + m_bytecodeGenerator.nextValueProfileIndex(); + + // Emit save sequence. + rewriter.insertFragmentBefore(instruction, [&] (BytecodeRewriter::Fragment& fragment) { + fragment.appendInstruction(scope, liveLocalsIndex, savedLocalsIndex, firstScopeOffset); + // Insert op_ret just after save sequence. + fragment.appendInstruction(data.argument); + }); + + // Emit resume sequence. + rewriter.replaceBytecodeWithFragment(instruction, [&] (BytecodeRewriter::Fragment& fragment) { + fragment.appendInstruction(scope, liveLocalsIndex, savedLocalsIndex, firstScopeOffset, firstValueProfile); + }); + } +} + +void BytecodeGeneratorification::emitPerLocalSaveAndRestore(BytecodeRewriter& rewriter) +{ + VM& vm = m_bytecodeGenerator.vm(); for (const YieldData& data : m_yields) { VirtualRegister scope = virtualRegisterForArgumentIncludingThis(static_cast(JSGenerator::Argument::Frame)); @@ -273,19 +359,6 @@ void BytecodeGeneratorification::run() }); }); } - - if (m_generatorFrameData) { - auto instruction = m_instructions.at(m_generatorFrameData->m_point); - rewriter.replaceBytecodeWithFragment(instruction, [&] (BytecodeRewriter::Fragment& fragment) { - if (!m_generatorFrameSymbolTable->scopeSize()) { - // This will cause us to put jsUndefined() into the generator frame's scope value. - fragment.appendInstruction(m_generatorFrameData->m_dst, m_generatorFrameData->m_initialValue); - } else - fragment.appendInstruction(m_generatorFrameData->m_dst, m_generatorFrameData->m_scope, m_generatorFrameData->m_symbolTable, m_generatorFrameData->m_initialValue); - }); - } - - rewriter.execute(); } void performGeneratorification(BytecodeGenerator& bytecodeGenerator, UnlinkedCodeBlockGenerator* codeBlock, JSInstructionStreamWriter& instructions, SymbolTable* generatorFrameSymbolTable, int generatorFrameSymbolTableIndex) diff --git a/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.h b/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.h index c9f1a9e5165b..9c0fa702dc84 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.h +++ b/Source/JavaScriptCore/bytecode/BytecodeGeneratorification.h @@ -26,12 +26,13 @@ #pragma once +#include + namespace JSC { class BytecodeGenerator; class SymbolTable; class UnlinkedCodeBlockGenerator; -class SymbolTable; struct JSOpcodeTraits; template struct BaseInstruction; @@ -41,4 +42,15 @@ using JSInstructionStreamWriter = InstructionStreamWriter; void performGeneratorification(BytecodeGenerator&, UnlinkedCodeBlockGenerator*, JSInstructionStreamWriter&, SymbolTable* generatorFrameSymbolTable, int generatorFrameSymbolTableIndex); +template +void forEachLiveGeneratorLocal(const BitVector& liveLocals, const BitVector& savedLocals, const Functor& functor) +{ + unsigned slot = 0; + savedLocals.forEachSetBit([&](size_t index) { + if (liveLocals.quickGet(index)) + functor(index, slot); + ++slot; + }); +} + } // namespace JSC diff --git a/Source/JavaScriptCore/bytecode/BytecodeList.rb b/Source/JavaScriptCore/bytecode/BytecodeList.rb index 08ad777dbbd9..eaf91e96ff26 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeList.rb +++ b/Source/JavaScriptCore/bytecode/BytecodeList.rb @@ -1176,6 +1176,23 @@ argument: VirtualRegister, } +op :save_generator_locals, + args: { + scope: VirtualRegister, + liveLocals: unsigned, + savedLocals: unsigned, + firstScopeOffset: unsigned, + } + +op :restore_generator_locals, + args: { + scope: VirtualRegister, + liveLocals: unsigned, + savedLocals: unsigned, + firstScopeOffset: unsigned, + valueProfile: unsigned, + } + op :check_traps op :log_shadow_chicken_prologue, diff --git a/Source/JavaScriptCore/bytecode/BytecodeUseDef.cpp b/Source/JavaScriptCore/bytecode/BytecodeUseDef.cpp index 256f127a094d..21dd84a64f39 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeUseDef.cpp +++ b/Source/JavaScriptCore/bytecode/BytecodeUseDef.cpp @@ -292,6 +292,8 @@ void computeUsesForBytecodeIndexImpl(const JSInstruction* instruction, Checkpoin USES(OpPutInternalField, base, value) USES(OpYield, argument) + USES(OpSaveGeneratorLocals, scope) + USES(OpRestoreGeneratorLocals, scope) USES(OpEnumeratorNext, mode, index, base, enumerator) USES(OpEnumeratorGetByVal, base, mode, propertyName, index, enumerator) @@ -459,6 +461,8 @@ void computeDefsForBytecodeIndexImpl(unsigned numVars, const JSInstruction* inst case op_log_shadow_chicken_prologue: case op_log_shadow_chicken_tail: case op_yield: + case op_save_generator_locals: + case op_restore_generator_locals: case op_nop: case op_unreachable: case op_super_sampler_begin: diff --git a/Source/JavaScriptCore/bytecode/BytecodeUseDef.h b/Source/JavaScriptCore/bytecode/BytecodeUseDef.h index 921e9812befb..36873f302248 100644 --- a/Source/JavaScriptCore/bytecode/BytecodeUseDef.h +++ b/Source/JavaScriptCore/bytecode/BytecodeUseDef.h @@ -43,12 +43,26 @@ void computeUsesForBytecodeIndex(Block* codeBlock, const JSInstruction* instruct functor(codeBlock->scopeRegister()); computeUsesForBytecodeIndexImpl(instruction, checkpoint, functor); + + if (opcodeID == op_save_generator_locals) { + auto bytecode = instruction->as(); + codeBlock->bitVector(bytecode.m_liveLocals).forEachSetBit([&](size_t index) { + functor(virtualRegisterForLocal(index)); + }); + } } template void computeDefsForBytecodeIndex(Block* codeBlock, const JSInstruction* instruction, Checkpoint checkpoint, const Functor& functor) { computeDefsForBytecodeIndexImpl(codeBlock->numVars(), instruction, checkpoint, functor); + + if (instruction->opcodeID() == op_restore_generator_locals) { + auto bytecode = instruction->as(); + codeBlock->bitVector(bytecode.m_liveLocals).forEachSetBit([&](size_t index) { + functor(virtualRegisterForLocal(index)); + }); + } } #undef CALL_FUNCTOR diff --git a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp index 983da967562c..8a0599797b99 100644 --- a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp +++ b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp @@ -34,6 +34,7 @@ #include "BooleanConstructor.h" #include "BuiltinNames.h" #include "BytecodeGenerator.h" +#include "BytecodeGeneratorification.h" #include "BytecodeOperandsForCheckpoint.h" #include "CacheableIdentifierInlines.h" #include "CallLinkStatus.h" @@ -10854,6 +10855,37 @@ void ByteCodeParser::parseBlock(unsigned limit) NEXT_OPCODE(op_create_lexical_environment); } + case op_save_generator_locals: { + auto bytecode = currentInstruction->as(); + CodeBlock* codeBlock = m_inlineStackTop->m_codeBlock; + Node* scopeNode = get(bytecode.m_scope); + forEachLiveGeneratorLocal(codeBlock->bitVector(bytecode.m_liveLocals), codeBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned slot) { + addToGraph(PutClosureVar, OpInfo(bytecode.m_firstScopeOffset + slot), scopeNode, get(virtualRegisterForLocal(index))); + emitExitOK(); + }); + addToGraph(Phantom, scopeNode); + NEXT_OPCODE(op_save_generator_locals); + } + + case op_restore_generator_locals: { + auto bytecode = currentInstruction->as(); + CodeBlock* codeBlock = m_inlineStackTop->m_codeBlock; + Node* scopeNode = get(bytecode.m_scope); + addToGraph(Phantom, scopeNode); + CodeBlock* profiledBlock = m_inlineStackTop->m_profiledBlock; + unsigned valueProfile = bytecode.m_valueProfile; + forEachLiveGeneratorLocal(codeBlock->bitVector(bytecode.m_liveLocals), codeBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned slot) { + VirtualRegister local = virtualRegisterForLocal(index); + SpeculatedType prediction = profiledBlock->valueProfileForOffset(valueProfile++).computeUpdatedPrediction(); + mergeSpeculation(prediction, m_inlineStackTop->m_lazyOperands.prediction(LazyOperandValueProfileKey(m_currentIndex, m_inlineStackTop->remapOperand(local)))); + if (prediction == SpecNone) + prediction = SpecEmpty; + set(local, addToGraph(GetClosureVar, OpInfo(bytecode.m_firstScopeOffset + slot), OpInfo(prediction), scopeNode), ImmediateSetWithFlush); + emitExitOK(); + }); + NEXT_OPCODE(op_restore_generator_locals); + } + case op_push_with_scope: { auto bytecode = currentInstruction->as(); Node* currentScope = get(bytecode.m_currentScope); diff --git a/Source/JavaScriptCore/dfg/DFGGraph.cpp b/Source/JavaScriptCore/dfg/DFGGraph.cpp index c990c53e8faf..71f0ee14596a 100644 --- a/Source/JavaScriptCore/dfg/DFGGraph.cpp +++ b/Source/JavaScriptCore/dfg/DFGGraph.cpp @@ -29,6 +29,7 @@ #if ENABLE(DFG_JIT) #include "ArrayPrototype.h" +#include "BytecodeGeneratorification.h" #include "CacheableIdentifierInlines.h" #include "CodeBlock.h" #include "CodeBlockWithJITType.h" @@ -1953,6 +1954,22 @@ MethodOfGettingAValueProfile Graph::methodOfGettingAValueProfileFor(Node* curren } case op_call_ignore_result: return { }; + case op_restore_generator_locals: { + if (node->op() != GetClosureVar) + return { }; + auto bytecode = instruction->as(); + unsigned slot = node->scopeOffset().offset() - bytecode.m_firstScopeOffset; + std::optional local; + forEachLiveGeneratorLocal(profiledBlock->bitVector(bytecode.m_liveLocals), profiledBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned candidate) { + if (candidate == slot) + local = virtualRegisterForLocal(index); + }); + if (!local) + return { }; + if (InlineCallFrame* inlineCallFrame = node->origin.semantic.inlineCallFrame()) + *local += inlineCallFrame->stackOffset; + return MethodOfGettingAValueProfile::lazyOperandValueProfile(node->origin.semantic, *local); + } default: { auto* valueProfile = profiledBlock->tryGetValueProfileForBytecodeIndex(node->origin.semantic.bytecodeIndex()); if (!valueProfile) diff --git a/Source/JavaScriptCore/jit/JIT.cpp b/Source/JavaScriptCore/jit/JIT.cpp index 4ea7e1ff89a9..054306ae475a 100644 --- a/Source/JavaScriptCore/jit/JIT.cpp +++ b/Source/JavaScriptCore/jit/JIT.cpp @@ -393,6 +393,8 @@ void JIT::privateCompileMainPass() DEFINE_OP(op_not) DEFINE_OP(op_nstricteq) DEFINE_OP(op_create_lexical_environment) + DEFINE_OP(op_save_generator_locals) + DEFINE_OP(op_restore_generator_locals) DEFINE_OP(op_create_direct_arguments) DEFINE_OP(op_create_scoped_arguments) DEFINE_OP(op_create_cloned_arguments) diff --git a/Source/JavaScriptCore/jit/JIT.h b/Source/JavaScriptCore/jit/JIT.h index 12906d97830b..555c8a76caf9 100644 --- a/Source/JavaScriptCore/jit/JIT.h +++ b/Source/JavaScriptCore/jit/JIT.h @@ -298,6 +298,7 @@ namespace JSC { void emitWriteBarrier(JSCell* owner); void emitWriteBarrier(GPRReg owner); + void emitValueProfilingSite(unsigned profileOffset, JSValueRegs); template void emitValueProfilingSite(const Bytecode&, JSValueRegs); template void emitValueProfilingSite(const Bytecode&, BytecodeIndex, JSValueRegs); @@ -477,6 +478,8 @@ namespace JSC { void emit_op_new_object(const JSInstruction*); void emit_op_new_reg_exp(const JSInstruction*); void emit_op_create_lexical_environment(const JSInstruction*); + void emit_op_save_generator_locals(const JSInstruction*); + void emit_op_restore_generator_locals(const JSInstruction*); void emit_op_create_direct_arguments(const JSInstruction*); void emit_op_create_scoped_arguments(const JSInstruction*); void emit_op_create_cloned_arguments(const JSInstruction*); diff --git a/Source/JavaScriptCore/jit/JITInlines.h b/Source/JavaScriptCore/jit/JITInlines.h index 764a0554e64b..952263f94db9 100644 --- a/Source/JavaScriptCore/jit/JITInlines.h +++ b/Source/JavaScriptCore/jit/JITInlines.h @@ -308,16 +308,21 @@ ALWAYS_INLINE bool JIT::isOperandConstantChar(VirtualRegister src) return getConstantOperand(src).isString() && asString(getConstantOperand(src).asCell())->length() == 1; } -template -inline void JIT::emitValueProfilingSite(const Bytecode& bytecode, BytecodeIndex bytecodeIndex, JSValueRegs value) +inline void JIT::emitValueProfilingSite(unsigned profileOffset, JSValueRegs value) { if (!shouldEmitProfiling()) return; - ptrdiff_t offset = -static_cast(valueProfileOffsetFor(bytecode, bytecodeIndex.checkpoint())) * sizeof(ValueProfile) + ValueProfile::offsetOfFirstBucket() - sizeof(UnlinkedMetadataTable::LinkingData); + ptrdiff_t offset = -static_cast(profileOffset) * sizeof(ValueProfile) + ValueProfile::offsetOfFirstBucket() - sizeof(UnlinkedMetadataTable::LinkingData); storeValue(value, Address(GPRInfo::metadataTableRegister, offset)); } +template +inline void JIT::emitValueProfilingSite(const Bytecode& bytecode, BytecodeIndex bytecodeIndex, JSValueRegs value) +{ + emitValueProfilingSite(valueProfileOffsetFor(bytecode, bytecodeIndex.checkpoint()), value); +} + template inline void JIT::emitValueProfilingSite(const Bytecode& bytecode, JSValueRegs value) { diff --git a/Source/JavaScriptCore/jit/JITOpcodes.cpp b/Source/JavaScriptCore/jit/JITOpcodes.cpp index 9edc1dce6060..4f340216c149 100644 --- a/Source/JavaScriptCore/jit/JITOpcodes.cpp +++ b/Source/JavaScriptCore/jit/JITOpcodes.cpp @@ -31,12 +31,14 @@ #include "BaselineJITRegisters.h" #include "BasicBlockLocation.h" #include "BinarySwitch.h" +#include "BytecodeGeneratorification.h" #include "BytecodeGenerator.h" #include "Exception.h" #include "JITInlines.h" #include "JITThunks.h" #include "JSCast.h" #include "JSFunction.h" +#include "JSLexicalEnvironment.h" #include "JSPropertyNameEnumerator.h" #include "JumpTable.h" #include "LinkBuffer.h" @@ -1970,6 +1972,37 @@ void JIT::emit_op_create_lexical_environment(const JSInstruction* currentInstruc callOperationNoExceptionCheck(value == jsUndefined() ? operationCreateLexicalEnvironmentUndefined : operationCreateLexicalEnvironmentTDZ, dst, argumentGPR0, argumentGPR1, argumentGPR2); } +void JIT::emit_op_save_generator_locals(const JSInstruction* currentInstruction) +{ + auto bytecode = currentInstruction->as(); + constexpr GPRReg scopeGPR = regT2; + constexpr JSValueRegs valueJSR = jsRegT10; + static_assert(noOverlap(scopeGPR, valueJSR)); + + emitGetVirtualRegister(bytecode.m_scope, scopeGPR); + forEachLiveGeneratorLocal(m_unlinkedCodeBlock->bitVector(bytecode.m_liveLocals), m_unlinkedCodeBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned slot) { + loadValue(addressFor(virtualRegisterForLocal(index)), valueJSR); + storeValue(valueJSR, Address(scopeGPR, JSLexicalEnvironment::offsetOfVariable(ScopeOffset(bytecode.m_firstScopeOffset + slot)))); + }); + emitWriteBarrier(scopeGPR); +} + +void JIT::emit_op_restore_generator_locals(const JSInstruction* currentInstruction) +{ + auto bytecode = currentInstruction->as(); + constexpr GPRReg scopeGPR = regT2; + constexpr JSValueRegs valueJSR = jsRegT10; + static_assert(noOverlap(scopeGPR, valueJSR)); + + emitGetVirtualRegister(bytecode.m_scope, scopeGPR); + unsigned valueProfile = bytecode.m_valueProfile; + forEachLiveGeneratorLocal(m_unlinkedCodeBlock->bitVector(bytecode.m_liveLocals), m_unlinkedCodeBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned slot) { + loadValue(Address(scopeGPR, JSLexicalEnvironment::offsetOfVariable(ScopeOffset(bytecode.m_firstScopeOffset + slot))), valueJSR); + storeValue(valueJSR, addressFor(virtualRegisterForLocal(index))); + emitValueProfilingSite(valueProfile++, valueJSR); + }); +} + void JIT::emit_op_create_direct_arguments(const JSInstruction* currentInstruction) { auto bytecode = currentInstruction->as(); diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter.asm index 2a8f2dd663bf..61ce41bca84d 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreter.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreter.asm @@ -2282,6 +2282,8 @@ slowPathOp(unreachable) slowPathOp(new_promise) slowPathOp(new_generator) slowPathOp(new_async_function_generator) +slowPathOp(save_generator_locals) +slowPathOp(restore_generator_locals) macro llintSlowPathOp(opcodeName) llintOp(op_%opcodeName%, unused, macro (unused, unused, dispatch) diff --git a/Source/JavaScriptCore/lol/LOLJIT.cpp b/Source/JavaScriptCore/lol/LOLJIT.cpp index 8b5f990d0396..0dcf3551ea0a 100644 --- a/Source/JavaScriptCore/lol/LOLJIT.cpp +++ b/Source/JavaScriptCore/lol/LOLJIT.cpp @@ -347,6 +347,8 @@ void LOLJIT::privateCompileMainPass() DEFINE_SLOW_OP(create_async_generator) DEFINE_SLOW_OP(new_generator) DEFINE_SLOW_OP(new_async_function_generator) + DEFINE_SLOW_OP(save_generator_locals) + DEFINE_SLOW_OP(restore_generator_locals) DEFINE_OP(op_add) DEFINE_OP(op_bitnot) diff --git a/Source/JavaScriptCore/runtime/CommonSlowPaths.cpp b/Source/JavaScriptCore/runtime/CommonSlowPaths.cpp index 3557e2387c8f..1e55d2ee8062 100644 --- a/Source/JavaScriptCore/runtime/CommonSlowPaths.cpp +++ b/Source/JavaScriptCore/runtime/CommonSlowPaths.cpp @@ -28,6 +28,7 @@ #include "ArithProfile.h" #include "ArrayPrototypeInlines.h" +#include "BytecodeGeneratorification.h" #include "BytecodeStructs.h" #include "ClonedArguments.h" #include "CommonSlowPathsInlines.h" @@ -1440,6 +1441,34 @@ JSC_DEFINE_COMMON_SLOW_PATH(slow_path_create_rest) RETURN(constructArray(globalObject, structure, argumentsToCopyRegion, argumentCount > numParamsToSkip ? argumentCount - numParamsToSkip : 0)); } +JSC_DEFINE_COMMON_SLOW_PATH(slow_path_save_generator_locals) +{ + BEGIN(); + auto bytecode = pc->as(); + JSLexicalEnvironment* environment = uncheckedDowncast(GET(bytecode.m_scope).jsValue().asCell()); + WriteBarrierBase* variables = environment->variables() + bytecode.m_firstScopeOffset; + forEachLiveGeneratorLocal(codeBlock->bitVector(bytecode.m_liveLocals), codeBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned slot) { + variables[slot].setWithoutWriteBarrier(GET(virtualRegisterForLocal(index)).jsValue()); + }); + vm.writeBarrier(environment); + END(); +} + +JSC_DEFINE_COMMON_SLOW_PATH(slow_path_restore_generator_locals) +{ + BEGIN(); + auto bytecode = pc->as(); + JSLexicalEnvironment* environment = uncheckedDowncast(GET(bytecode.m_scope).jsValue().asCell()); + WriteBarrierBase* variables = environment->variables() + bytecode.m_firstScopeOffset; + unsigned valueProfile = bytecode.m_valueProfile; + forEachLiveGeneratorLocal(codeBlock->bitVector(bytecode.m_liveLocals), codeBlock->bitVector(bytecode.m_savedLocals), [&](size_t index, unsigned slot) { + JSValue value = variables[slot].get(); + GET(virtualRegisterForLocal(index)) = value; + codeBlock->valueProfileForOffset(valueProfile++).m_buckets[0] = JSValue::encode(value); + }); + END(); +} + JSC_DEFINE_COMMON_SLOW_PATH(slow_path_get_by_val_with_this) { BEGIN(); diff --git a/Source/JavaScriptCore/runtime/CommonSlowPaths.h b/Source/JavaScriptCore/runtime/CommonSlowPaths.h index 3d70e094a4d6..9673cd797a0a 100644 --- a/Source/JavaScriptCore/runtime/CommonSlowPaths.h +++ b/Source/JavaScriptCore/runtime/CommonSlowPaths.h @@ -315,6 +315,8 @@ JSC_DECLARE_COMMON_SLOW_PATH(slow_path_create_promise); JSC_DECLARE_COMMON_SLOW_PATH(slow_path_create_generator); JSC_DECLARE_COMMON_SLOW_PATH(slow_path_create_async_generator); JSC_DECLARE_COMMON_SLOW_PATH(slow_path_create_rest); +JSC_DECLARE_COMMON_SLOW_PATH(slow_path_save_generator_locals); +JSC_DECLARE_COMMON_SLOW_PATH(slow_path_restore_generator_locals); JSC_DECLARE_COMMON_SLOW_PATH(slow_path_get_by_val_with_this); JSC_DECLARE_COMMON_SLOW_PATH(slow_path_get_prototype_of); JSC_DECLARE_COMMON_SLOW_PATH(slow_path_put_by_id_with_this); diff --git a/Source/JavaScriptCore/runtime/OptionsList.h b/Source/JavaScriptCore/runtime/OptionsList.h index 01a97e6a5e35..f14418438d9f 100644 --- a/Source/JavaScriptCore/runtime/OptionsList.h +++ b/Source/JavaScriptCore/runtime/OptionsList.h @@ -341,6 +341,7 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, breakOnThrow, false, Normal, nullptr) \ \ v(Unsigned, maximumOptimizationCandidateBytecodeCost, 100000, Normal, nullptr) \ + v(Bool, useGeneratorBulkSaveRestore, true, Normal, "Save and restore generator locals with bulk bytecodes instead of per-local scope ops."_s) \ v(Unsigned, maximumCachedAssemblerBufferSize, 1 * MB, Normal, "Assembler scratch buffers larger than this are freed after compilation instead of being cached per thread (0 = cache any size)"_s) \ \ v(Unsigned, maximumFunctionForCallInlineCandidateBytecodeCostForDFG, 80, Normal, nullptr) \