diff --git a/libwild/src/elf_writer.rs b/libwild/src/elf_writer.rs index 654c8ab19..f54db1bd2 100644 --- a/libwild/src/elf_writer.rs +++ b/libwild/src/elf_writer.rs @@ -84,6 +84,7 @@ use crate::platform::SectionHeader as _; use crate::resolution::SectionSlot; use crate::sframe; use crate::sharding::ShardKey; +use crate::string_merging::MergedStringTarget; use crate::string_merging::get_merged_string_output_address; use crate::symbol_db::SymbolDb; use crate::symbol_db::SymbolId; @@ -2749,6 +2750,109 @@ struct DebugRelocationShardSummary { max_write_end: u64, } +/// How an absolute debug relocation against a symbol gets its value, decided once per symbol so +/// the per-relocation work in [`try_apply_debug_relocation_fast`] is an add and a store. +#[derive(Clone, Copy, Debug)] +enum DebugFastValue { + /// The value is this base plus the addend. + Base(u64), + /// The symbol's section was discarded or never loaded, so the value is the tombstone. + Tombstone, + /// An unnamed symbol in a string-merge section, such as `.debug_str`'s section symbol. The + /// addend picks the string. + MergedString(MergedStringTarget), + /// Needs the full path: merged strings, ifuncs, and anything that might report an error. + Slow, +} + +/// How to write an absolute debug relocation of one type, extracted from its +/// [`RelocationKindInfo`]: a plain little-endian store of `size` bytes after a range check. +#[derive(Clone, Copy)] +struct DebugAbsoluteWrite { + r_type: object::elf::RelocationType, + size: u8, + range: linker_utils::elf::AllowedRange, +} + +impl DebugAbsoluteWrite { + /// Returns None for relocation types that need more than a plain store, which then take the + /// slow path. + fn new(r_type: object::elf::RelocationType, info: &RelocationKindInfo) -> Option { + let RelocationSize::ByteSize(size @ (4 | 8)) = info.size else { + return None; + }; + (info.kind == RelocationKind::Absolute && info.alignment == 1).then_some(Self { + r_type, + size, + range: info.range, + }) + } + + #[inline(always)] + fn write(self, value: u64, out: &mut [u8]) -> bool { + if !self.range.contains(value as i64) { + return false; + } + // Fixed-size arms, so that each is a single store rather than a variable-length copy. + let written = match self.size { + 4 => out + .first_chunk_mut() + .map(|place: &mut [u8; 4]| *place = (value as u32).to_le_bytes()), + 8 => out + .first_chunk_mut() + .map(|place: &mut [u8; 8]| *place = value.to_le_bytes()), + _ => None, + }; + written.is_some() + } +} + +/// Recently used [`DebugAbsoluteWrite`]s, indexed by the low bit of the relocation type. DWARF +/// alternates between a 32-bit and a 64-bit absolute type, and on x86-64 and AArch64 those have +/// different low bits, so both stay cached. Entries are checked against the type, so types that +/// share a slot only cost cache misses. +type DebugWriteCache = [Option; 2]; + +/// Number of entries in [`DebugSymbolCache`]. A power of two, so the slot is the low bits of the +/// symbol index. +const DEBUG_SYMBOL_CACHE_SLOTS: usize = 32; + +/// A direct-mapped cache of symbol resolutions for debug relocations, local to one relocation +/// shard. `.debug_info` interleaves references to a few hot symbols (the `.debug_str` and +/// `.debug_line` section symbols) with references to code, so a single "previous symbol" entry +/// misses about half the time. A few dozen slots catch nearly all of the repeats. The slots live +/// on the stack, because a cache is created for every debug section and most sections are small. +struct DebugSymbolCache { + slots: [Option<(object::SymbolIndex, DebugFastValue)>; DEBUG_SYMBOL_CACHE_SLOTS], +} + +impl Default for DebugSymbolCache { + fn default() -> Self { + Self { + slots: [None; DEBUG_SYMBOL_CACHE_SLOTS], + } + } +} + +impl DebugSymbolCache { + #[inline(always)] + fn get_or_insert_with( + &mut self, + symbol_index: object::SymbolIndex, + resolve: impl FnOnce() -> Result, + ) -> Result { + let slot = &mut self.slots[symbol_index.0 % DEBUG_SYMBOL_CACHE_SLOTS]; + if let Some((cached_index, value)) = *slot + && cached_index == symbol_index + { + return Ok(value); + } + let value = resolve()?; + *slot = Some((symbol_index, value)); + Ok(value) + } +} + fn debug_relocation_write_size< C: ElfClass, A: Arch>, @@ -3022,6 +3126,8 @@ fn apply_debug_relocations_impl< previous, ..Default::default() }; + let mut symbol_cache = DebugSymbolCache::default(); + let mut write_cache = DebugWriteCache::default(); for rel in relocations { relocation_count += 1; @@ -3030,6 +3136,19 @@ fn apply_debug_relocations_impl< let shard_offset = offset_in_section .checked_sub(output_offset) .context("Debug relocation precedes its output shard")?; + if try_apply_debug_relocation_fast::( + object, + shard_offset, + &rel, + layout, + tombstone_value, + out, + &mut symbol_cache, + &mut write_cache, + ) { + relocation_cache.previous = Some(rel); + continue; + } apply_debug_relocation::( object, shard_offset, @@ -3068,7 +3187,9 @@ fn record_debug_relocations( #[cfg(test)] mod debug_relocation_shard_tests { + use super::DebugFastValue; use super::DebugRelocationShardRange; + use super::DebugSymbolCache; use super::debug_relocation_shard_ranges_parallel; #[test] @@ -3173,6 +3294,37 @@ mod debug_relocation_shard_tests { assert_eq!(ranges.len(), 4); assert_eq!(size_queries.load(std::sync::atomic::Ordering::Relaxed), 12); } + + #[test] + fn debug_symbol_cache_reuses_recent_symbols() { + let resolve_count = std::cell::Cell::new(0); + let mut cache = DebugSymbolCache::default(); + let mut get = |symbol_index: usize| { + let value = cache + .get_or_insert_with(object::SymbolIndex(symbol_index), || { + resolve_count.set(resolve_count.get() + 1); + Ok(DebugFastValue::Base(symbol_index as u64)) + }) + .unwrap(); + let DebugFastValue::Base(base) = value else { + panic!("Unexpected cached value {value:?}"); + }; + base + }; + + assert_eq!(get(7), 7); + assert_eq!(get(7), 7); + assert_eq!(get(8), 8); + // 7 and 8 occupy different slots, so returning to 7 is a hit. + assert_eq!(get(7), 7); + assert_eq!(resolve_count.get(), 2); + + // A symbol that maps to 7's slot evicts it. + let colliding = 7 + super::DEBUG_SYMBOL_CACHE_SLOTS; + assert_eq!(get(colliding), colliding as u64); + assert_eq!(get(7), 7); + assert_eq!(resolve_count.get(), 4); + } } fn write_eh_frame_data<'data, C: ElfClass, A: Arch>>( @@ -4310,29 +4462,16 @@ fn maybe_get_thunk_for_relocation>>( ); } -fn apply_debug_relocation< - 'data, - C: ElfClass, - A: Arch>, - R: Relocation>, ->( +/// The resolution used for a symbol referenced from a debug section: the merged resolution if the +/// symbol has one, otherwise the address of the symbol within its section. +fn debug_symbol_resolution<'data, C: ElfClass>( object_layout: &ObjectLayout<'data, elf::Elf>, - offset_in_section: u64, - rel: &R, + sym: &elf::SymtabEntry, + symbol_index: object::SymbolIndex, + section_index: Option, layout: &ElfLayout, - section_tombstone_value: u64, - out: &mut [u8], - relocation_cache: &RelocationCache, -) -> Result<()> { - let symbol_index = rel.symbol().context("Unsupported absolute relocation")?; - let sym = object_layout.object.symbol(symbol_index)?; - let section_index = object_layout.object.symbol_section(sym, symbol_index)?; - - let addend = rel.addend(); - let r_type = rel.raw_type(); - let rel_info = A::relocation_from_raw(r_type)?; - - let resolution = layout +) -> Option>> { + layout .merged_symbol_resolution(object_layout.symbol_id_range.input_to_id(symbol_index)) .or_else(|| { section_index.and_then(|section_index| { @@ -4355,7 +4494,144 @@ fn apply_debug_relocation< format_specific: Default::default(), }) }) - }); + }) +} + +/// Decides how absolute debug relocations against `symbol_index` get their value. This must agree +/// with [`apply_debug_relocation`], which handles everything that returns +/// [`DebugFastValue::Slow`]. +fn debug_fast_value<'data, C: ElfClass>( + object_layout: &ObjectLayout<'data, elf::Elf>, + symbol_index: object::SymbolIndex, + layout: &ElfLayout, +) -> Result { + let sym = object_layout.object.symbol(symbol_index)?; + let section_index = object_layout.object.symbol_section(sym, symbol_index)?; + let resolution = + debug_symbol_resolution(object_layout, sym, symbol_index, section_index, layout); + + let in_merge_strings = section_index.is_some_and(|section_index| { + matches!( + object_layout.sections[section_index.0], + SectionSlot::MergeStrings(..) + ) + }); + let merged_string = || -> Result { + Ok(MergedStringTarget::for_unnamed_symbol::>( + symbol_index, + object_layout.object, + &object_layout.sections, + &layout.symbol_db.section_part_ids, + object_layout.section_id_range, + )? + .map_or(DebugFastValue::Slow, DebugFastValue::MergedString)) + }; + // `Resolution::value_with_addend` only looks up a merged string when `raw_value` is zero and + // the symbol is in a merge-strings section, and an unresolved symbol in a merge-strings + // section takes the same lookup. + Ok(match resolution { + Some(resolution) if resolution.flags.is_ifunc() => DebugFastValue::Slow, + Some(resolution) if resolution.raw_value == 0 && in_merge_strings => merged_string()?, + Some(resolution) => DebugFastValue::Base(resolution.raw_value), + None => match section_index { + None => DebugFastValue::Tombstone, + Some(section_index) => match object_layout.sections[section_index.0] { + SectionSlot::Discard | SectionSlot::Unloaded(..) => DebugFastValue::Tombstone, + SectionSlot::MergeStrings(..) => merged_string()?, + _ => DebugFastValue::Slow, + }, + }, + }) +} + +/// Applies an absolute debug relocation whose value is a base plus addend or a tombstone, which +/// covers nearly all DWARF relocations. Returns false, having written nothing, for anything else +/// or anything that fails, so [`apply_debug_relocation`] handles it and reports any error. +#[inline(always)] +fn try_apply_debug_relocation_fast< + 'data, + C: ElfClass, + A: Arch>, + R: Relocation>, +>( + object_layout: &ObjectLayout<'data, elf::Elf>, + offset_in_section: u64, + rel: &R, + layout: &ElfLayout, + section_tombstone_value: u64, + out: &mut [u8], + symbol_cache: &mut DebugSymbolCache, + write_cache: &mut DebugWriteCache, +) -> bool { + let Some(symbol_index) = rel.symbol() else { + return false; + }; + let r_type = rel.raw_type(); + let slot = &mut write_cache[(r_type.0 & 1) as usize]; + let plan = match *slot { + Some(plan) if plan.r_type == r_type => plan, + _ => { + let Some(plan) = A::relocation_from_raw(r_type) + .ok() + .and_then(|info| DebugAbsoluteWrite::new(r_type, &info)) + else { + return false; + }; + *slot = Some(plan); + plan + } + }; + let Ok(fast_value) = symbol_cache.get_or_insert_with(symbol_index, || { + debug_fast_value(object_layout, symbol_index, layout) + }) else { + return false; + }; + let value = match fast_value { + DebugFastValue::Base(base) => base.wrapping_add(rel.addend() as u64), + DebugFastValue::Tombstone => section_tombstone_value, + DebugFastValue::MergedString(target) => { + let Ok(address) = target.unnamed_address( + rel.addend(), + &layout.merged_strings, + &layout.merged_string_start_addresses, + ) else { + return false; + }; + address + } + DebugFastValue::Slow => return false, + }; + let Some(place) = out.get_mut(offset_in_section as usize..) else { + return false; + }; + plan.write(value, place) +} + +#[inline(never)] +fn apply_debug_relocation< + 'data, + C: ElfClass, + A: Arch>, + R: Relocation>, +>( + object_layout: &ObjectLayout<'data, elf::Elf>, + offset_in_section: u64, + rel: &R, + layout: &ElfLayout, + section_tombstone_value: u64, + out: &mut [u8], + relocation_cache: &RelocationCache, +) -> Result<()> { + let symbol_index = rel.symbol().context("Unsupported absolute relocation")?; + let sym = object_layout.object.symbol(symbol_index)?; + let section_index = object_layout.object.symbol_section(sym, symbol_index)?; + + let addend = rel.addend(); + let r_type = rel.raw_type(); + let rel_info = A::relocation_from_raw(r_type)?; + + let resolution = + debug_symbol_resolution(object_layout, sym, symbol_index, section_index, layout); let value = if let Some(resolution) = resolution { match rel_info.kind { diff --git a/libwild/src/string_merging.rs b/libwild/src/string_merging.rs index 2f622cb40..629cbad8f 100644 --- a/libwild/src/string_merging.rs +++ b/libwild/src/string_merging.rs @@ -33,6 +33,7 @@ use crate::error::Result; use crate::hash::PassThroughHashMap; use crate::hash::PreHashed; use crate::input_section_id::SectionIdRange; +use crate::output_section_id::OutputSectionId; use crate::output_section_id::OutputSections; use crate::output_section_map::OutputSectionMap; use crate::output_section_part_map::OutputSectionPartMap; @@ -1014,7 +1015,7 @@ pub(crate) fn get_merged_string_output_address<'data, P: Platform>( let SectionSlot::MergeStrings(merge_slot) = §ions[section_index.0] else { return Ok(None); }; - let mut input_offset = symbol.value(); + let input_offset = symbol.value(); // When we reference data in a string-merge section via a named symbol, we determine which // string we're referencing without taking the addend into account, then apply the addend @@ -1022,6 +1023,12 @@ pub(crate) fn get_merged_string_output_address<'data, P: Platform>( // addend into account up-front before we determine which string we're pointing at. This is a // bit weird, but seems to match what other linkers do. let symbol_has_name = symbol.has_name(); + let part_id = section_part_ids[input_section_id.as_usize()]; + let target = MergedStringTarget { + merge_slot: *merge_slot, + section_id: part_id.output_section_id::

(), + symbol_value: input_offset, + }; if !symbol_has_name { // We're computing a resolution for an unnamed symbol, just use the value of 0 for now. // We'll compute the address later when we're processing relocations that reference the @@ -1029,20 +1036,81 @@ pub(crate) fn get_merged_string_output_address<'data, P: Platform>( if zero_unnamed { return Ok(Some(0)); } - input_offset = input_offset.wrapping_add(addend as u64); + return target + .unnamed_address(addend, merged_strings, merged_string_start_addresses) + .map(Some); } - let part_id = section_part_ids[input_section_id.as_usize()]; - let section_id = part_id.output_section_id::

(); - let strings_section = merged_strings.get(section_id); - let string_offset = find_string(*merge_slot, input_offset, strings_section)?; - let bucket_base = - merged_string_start_addresses.addresses.get(section_id)[string_offset.bucket()]; - let mut address = bucket_base + string_offset.offset_in_bucket(); - if symbol_has_name { - address = address.wrapping_add(addend as u64); + let string_offset = find_string( + target.merge_slot, + input_offset, + merged_strings.get(target.section_id), + )?; + let bucket_base = merged_string_start_addresses + .addresses + .get(target.section_id)[string_offset.bucket()]; + Ok(Some( + (bucket_base + string_offset.offset_in_bucket()).wrapping_add(addend as u64), + )) +} + +/// The parts of a merged-string lookup that depend only on the symbol. Debug info references +/// `.debug_str` through its unnamed section symbol with a different addend for every string, so +/// computing these once per symbol leaves only the string lookup per relocation. +#[derive(Debug, Clone, Copy)] +pub(crate) struct MergedStringTarget { + merge_slot: StringMergeSectionSlot, + section_id: OutputSectionId, + symbol_value: u64, +} + +impl MergedStringTarget { + /// Returns the target for an unnamed symbol in a string-merge section, or None for a named + /// symbol or one in any other kind of section. + pub(crate) fn for_unnamed_symbol<'data, P: Platform>( + symbol_index: object::SymbolIndex, + object: &P::File<'data>, + sections: &[SectionSlot], + section_part_ids: &[PartId], + section_id_range: SectionIdRange, + ) -> Result> { + let symbol = object.symbol(symbol_index)?; + let Some(section_index) = object.symbol_section(symbol, symbol_index)? else { + return Ok(None); + }; + let SectionSlot::MergeStrings(merge_slot) = §ions[section_index.0] else { + return Ok(None); + }; + if symbol.has_name() { + return Ok(None); + } + let input_section_id = section_id_range.input_to_id(section_index); + Ok(Some(Self { + merge_slot: *merge_slot, + section_id: section_part_ids[input_section_id.as_usize()].output_section_id::

(), + symbol_value: symbol.value(), + })) + } + + /// The output address referenced by an unnamed symbol plus `addend`. The addend selects the + /// string, matching what other linkers do for section references. + #[inline(always)] + pub(crate) fn unnamed_address( + self, + addend: i64, + merged_strings: &OutputSectionMap, + merged_string_start_addresses: &MergedStringStartAddresses, + ) -> Result { + let input_offset = self.symbol_value.wrapping_add(addend as u64); + let string_offset = find_string( + self.merge_slot, + input_offset, + merged_strings.get(self.section_id), + )?; + let bucket_base = + merged_string_start_addresses.addresses.get(self.section_id)[string_offset.bucket()]; + Ok(bucket_base + string_offset.offset_in_bucket()) } - Ok(Some(address)) } fn find_string(