[AUTOMATED] Recover Go string records from writable data sections (fatstr Go arm)

decompiler/crates/kuna-analysis/src/analyzers/strings/kuna_strrecords.rs

@@ -36,12 +36,27 @@
 //!
 //! # Gating
 //!
-//! One option, two recognizers: this pass is registered only for a
-//! rustc-detected image (`passes_for`), and its commit is gated on the same
+//! One option, two recognizers: this pass is registered only for rustc- and
+//! Go-detected images (`passes_for`), and its commit is gated on the same
 //! `Architecture::fat_str` mode the engine rule reads (`off` drops the facts
-//! at the read-symbols boundary, exactly like every analysis gate). Go string
-//! headers share the record shape but belong to the Go pack's blob
-//! segmentation item, deliberately not here.
+//! at the read-symbols boundary, exactly like every analysis gate).
+//!
+//! # The Go arm
+//!
+//! Go string headers are the same `(ptr, len)` record over the same
+//! unterminated `.rodata` blob, with one placement difference: the measured
+//! go1.22 toolchain binaries are non-PIE and keep most headers in the
+//! WRITABLE data image (`.data`: 480 of 545 records on the hello witness;
+//! `.noptrdata` holds the pointer-free statics), with the remaining 65 in
+//! `.rodata` itself. A Go registration therefore widens the SLOT regions to
+//! those two sections by name ([`StrRecordPass::go`]); the TARGET test is
+//! unchanged — the text must sit in genuinely read-only data — which is what
+//! keeps a writable slot's mutability irrelevant (the record may move; the
+//! literal it currently states cannot). Go also shares literal bytes between
+//! headers (substring records): records are deduped longest-first per start
+//! address, and a record starting inside an earlier one is retired by the
+//! commit arm's occupied-address guard, so prefix/suffix sharing renders
+//! through the containing literal.
 
 use object::read::{Object, ObjectSection};
 use object::RelocationFlags;
@@ -89,6 +104,15 @@ fn ro_regions<'a>(file: &'a object::File) -> Vec<RoRegion<'a>> {
         .collect()
 }
 
+/// The additional SLOT regions a Go image keeps headers in: the writable data
+/// image. Slots only — [`record_at`] still demands the target bytes in RO.
+fn go_slot_regions<'a>(file: &'a object::File) -> Vec<RoRegion<'a>> {
+    file.sections()
+        .filter(|s| s.name().is_ok_and(|n| n == ".data" || n == ".noptrdata"))
+        .filter_map(|s| Some(RoRegion { vma: s.address(), data: s.data().ok()? }))
+        .collect()
+}
+
 /// Is this record text? Valid UTF-8 whose chars are >=90% printable or
 /// whitespace — the engine rule's text-evidence test, applied to exact bytes.
 fn is_text(bytes: &[u8]) -> bool {
@@ -163,18 +187,19 @@ fn scan_reloc_records(file: &object::File, regions: &[RoRegion], width: u64) ->
     out
 }
 
-/// The direct arm: absolute `(ptr, len)` word pairs in read-only data (the
-/// non-PIE layout; on a PIE the unrelocated slots read zero and match nothing).
-fn scan_direct_records(regions: &[RoRegion], width: u64) -> Vec<StringFact> {
+/// The direct arm: absolute `(ptr, len)` word pairs whose slots sit in
+/// `slots` and whose targets sit in `targets` (the non-PIE layout; on a PIE
+/// the unrelocated slots read zero and match nothing).
+fn scan_direct_records(slots: &[RoRegion], targets: &[RoRegion], width: u64) -> Vec<StringFact> {
     let mut out = Vec::new();
     let w = width as usize;
-    for region in regions {
+    for region in slots {
         let data = region.data;
         let mut i = 0usize;
         while i + 2 * w <= data.len() {
             let ptr = read_word(&data[i..i + w]);
             let len = read_word(&data[i + w..i + 2 * w]);
-            if let Some(fact) = record_at(regions, ptr, len, DIRECT_MIN_LEN) {
+            if let Some(fact) = record_at(targets, ptr, len, DIRECT_MIN_LEN) {
                 out.push(fact);
                 if out.len() >= MAX_RECORDS {
                     return out;
@@ -186,9 +211,24 @@ fn scan_direct_records(regions: &[RoRegion], width: u64) -> Vec<StringFact> {
     out
 }
 
-/// (kuna `fatstr`) The `&str`-record string pass. Registered for rustc-detected
-/// images; committed under the `fatstr` mode gate.
-pub struct StrRecordPass;
+/// (kuna `fatstr`) The string-record pass. Registered for rustc- and
+/// Go-detected images; committed under the `fatstr` mode gate.
+pub struct StrRecordPass {
+    /// Widen the direct arm's SLOT regions to the Go data image
+    /// ([`go_slot_regions`]); targets stay read-only either way.
+    go_slots: bool,
+}
+
+impl StrRecordPass {
+    /// The rustc registration: slots in read-only data only.
+    pub fn rust() -> StrRecordPass {
+        StrRecordPass { go_slots: false }
+    }
+    /// The Go registration: slots also in `.data`/`.noptrdata`.
+    pub fn go() -> StrRecordPass {
+        StrRecordPass { go_slots: true }
+    }
+}
 
 impl AnalysisPass for StrRecordPass {
     fn phase(&self) -> Phase {
@@ -204,11 +244,17 @@ impl AnalysisPass for StrRecordPass {
         }
         let width: u64 = if ctx.file.is_64() { 8 } else { 4 };
         let mut facts = scan_reloc_records(ctx.file, &regions, width);
-        facts.extend(scan_direct_records(&regions, width));
-        // One fact per start address; the first (reloc-vouched) wins, and a
-        // longest-first order inside equal starts is irrelevant because records
-        // state exact lengths.
-        facts.sort_by_key(|f| f.addr);
+        if self.go_slots {
+            let slots = go_slot_regions(ctx.file);
+            facts.extend(scan_direct_records(&slots, &regions, width));
+        }
+        facts.extend(scan_direct_records(&regions, &regions, width));
+        // One fact per start address, LONGEST first: Go headers share bytes
+        // (prefix records at one start), and the longer record renders the
+        // shorter as a prefix read while the reverse would truncate. A record
+        // starting inside an earlier one is left to the commit arm's
+        // occupied-address guard.
+        facts.sort_by_key(|f| (f.addr, core::cmp::Reverse(f.len)));
         facts.dedup_by_key(|f| f.addr);
         AnalysisOutput { bounded_strings: facts, ..Default::default() }
     }
@@ -264,7 +310,7 @@ mod tests {
         }
         data[0x100..0x100 + recs.len()].copy_from_slice(&recs);
         let regions = vec![region(0x2000, &data)];
-        let found = scan_direct_records(&regions, 8);
+        let found = scan_direct_records(&regions, &regions, 8);
         // The 17-byte record is found (once per slot it appears in); the 1-byte
         // record fails the direct-arm minimum. Dedup collapses the repeat.
         let mut facts = found;
@@ -273,4 +319,40 @@ mod tests {
         assert_eq!(facts.len(), 1);
         assert_eq!((facts[0].addr, facts[0].len), (0x2000, 17));
     }
+
+    /// The Go slot split: records whose SLOTS sit outside the target regions
+    /// (the `.data` header area) are found when passed as the slot set, and
+    /// the targets are still held to the read-only set.
+    #[test]
+    fn go_slots_scan_separately_from_targets() {
+        let mut ro = vec![0u8; 0x40];
+        ro[..17].copy_from_slice(b"hello multilang, ");
+        let targets = vec![region(0x2000, &ro)];
+        // Header area at 0x5000 ("data"): one record to the RO text, one
+        // record whose target points back into the header area itself.
+        let mut hdr = vec![0u8; 0x40];
+        hdr[..8].copy_from_slice(&0x2000u64.to_le_bytes());
+        hdr[8..16].copy_from_slice(&17u64.to_le_bytes());
+        hdr[16..24].copy_from_slice(&0x5000u64.to_le_bytes());
+        hdr[24..32].copy_from_slice(&17u64.to_le_bytes());
+        let slots = vec![region(0x5000, &hdr)];
+        let found = scan_direct_records(&slots, &targets, 8);
+        assert_eq!(found.len(), 1, "only the RO-backed target is a record");
+        assert_eq!((found[0].addr, found[0].len), (0x2000, 17));
+    }
+
+    /// Same-start records keep the LONGEST (Go prefix sharing): the shorter
+    /// renders as a prefix read of the longer.
+    #[test]
+    fn same_start_dedup_keeps_the_longest() {
+        let mut facts = vec![
+            StringFact { addr: 0x2000, len: 5 },
+            StringFact { addr: 0x2000, len: 17 },
+            StringFact { addr: 0x2020, len: 4 },
+        ];
+        facts.sort_by_key(|f| (f.addr, core::cmp::Reverse(f.len)));
+        facts.dedup_by_key(|f| f.addr);
+        assert_eq!(facts.len(), 2);
+        assert_eq!((facts[0].addr, facts[0].len), (0x2000, 17));
+    }
 }

decompiler/crates/kuna-analysis/src/passes.rs

@@ -394,6 +394,12 @@ pub fn passes_for(compiler: Compiler, format: object::BinaryFormat) -> Vec<Box<d
     // facts idempotently (a real `.symtab` name still wins, so only a *stripped*
     // Go binary's `sub_<addr>` functions take the recovered name).
     if compiler.is_golang() {
+        // (kuna `fatstr`) The Go arm of the string-record pass: the same
+        // (ptr, len) record shape rustc emits, with the slots widened to the
+        // Go data image (.data/.noptrdata). FIRST in the Go block so
+        // `gopclntab` keeps its appended-last contract. Committed under the
+        // same `fatstr` mode gate as the rustc arm.
+        passes.push(Box::new(crate::strings::kuna_strrecords::StrRecordPass::go()));
         // (kuna `gonoreturn`) BEFORE `gopclntab`, preserving that pass's
         // appended-last contract: apply the vendored Go no-return list to the
         // pclntab's own function list, so a STRIPPED Go binary (no .symtab for
@@ -409,7 +415,7 @@ pub fn passes_for(compiler: Compiler, format: object::BinaryFormat) -> Vec<Box<d
     // the `fatstr` mode gate (`Architecture::fat_str`), so `--option fatstr
     // off` drops the facts at the read-symbols boundary.
     if compiler.is_rust() {
-        passes.push(Box::new(crate::strings::kuna_strrecords::StrRecordPass));
+        passes.push(Box::new(crate::strings::kuna_strrecords::StrRecordPass::rust()));
     }
 
     // S1 PE/Mach-O import-slot call binding (PeImportCallPass): paint
@@ -1660,24 +1666,27 @@ mod tests {
         assert_eq!(&rust[..base.len()], &base[..], "Rust set is the base set + extras");
         assert_eq!(rust.last(), Some(&"fatstrrecords"), "Rust appends the record pass");
         assert_eq!(rust.len(), base.len() + 1, "Rust adds exactly one pass");
-        // No non-Rust compiler carries the record pass.
-        for c in [Compiler::Gcc, Compiler::Clang, Compiler::Go, Compiler::Swift, Compiler::Unknown] {
+        // No compiler without a string-record arm carries the record pass
+        // (Go carries it too — the next test).
+        for c in [Compiler::Gcc, Compiler::Clang, Compiler::Swift, Compiler::Unknown] {
             assert!(!ids(&passes_for(c, NON_PE)).contains(&"fatstrrecords"), "{c:?} must not carry fatstrrecords");
         }
     }
 
-    /// The Go pass set is exactly the base set plus the two Go-only passes:
-    /// `gonoreturn` (the pclntab no-return list) then `gopclntab` (the
-    /// function-name recovery, keeping its appended-last contract); no other
-    /// compiler carries either.
+    /// The Go pass set is exactly the base set plus the three Go-arm passes:
+    /// `fatstrrecords` (the string-record pass's Go arm), then `gonoreturn`
+    /// (the pclntab no-return list), then `gopclntab` (the function-name
+    /// recovery, keeping its appended-last contract); no other compiler
+    /// carries the two Go-only ones.
     #[test]
-    fn go_adds_only_the_two_go_passes() {
+    fn go_adds_only_the_three_go_passes() {
         let base = ids(&passes_for(Compiler::Gcc, NON_PE));
         let go = ids(&passes_for(Compiler::Go, NON_PE));
         assert_eq!(&go[..base.len()], &base[..], "Go set is the base set + extras");
         assert_eq!(go.last(), Some(&"gopclntab"), "Go appends the gopclntab pass last");
         assert_eq!(go.get(go.len() - 2), Some(&"gonoreturn"), "gonoreturn precedes gopclntab");
-        assert_eq!(go.len(), base.len() + 2, "Go adds exactly two passes");
+        assert_eq!(go.get(go.len() - 3), Some(&"fatstrrecords"), "the record pass opens the Go block");
+        assert_eq!(go.len(), base.len() + 3, "Go adds exactly three passes");
         // No non-Go compiler carries either Go-only pass.
         for c in [Compiler::Gcc, Compiler::Rustc, Compiler::Clang, Compiler::Swift, Compiler::Unknown] {
             let set = ids(&passes_for(c, NON_PE));

decompiler/crates/kuna-console/tests/verify_fatstr_records.rs

@@ -77,6 +77,57 @@ fn build_rust(dir: &std::path::Path, out_bin: &std::path::Path) -> bool {
         .unwrap_or(false)
 }
 
+/// The Go twin of [`MAIN_RS`]: the same runtime-indexed table, whose headers
+/// the Go linker emits as absolute `(ptr, len)` pairs in the data image.
+const MAIN_GO: &str = r#"package main
+
+import (
+	"fmt"
+	"os"
+)
+
+var table = [3]string{"kuna fatstr witness, ", "kuna second piece", "kuna third piece"}
+
+func main() {
+	i := len(os.Args) % 3
+	fmt.Println(table[i] + "!")
+}
+"#;
+
+/// `true` if a runnable `go` toolchain is on PATH.
+fn go_available() -> bool {
+    Command::new("go")
+        .arg("version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .map(|s| s.success())
+        .unwrap_or(false)
+}
+
+/// Build [`MAIN_GO`] stripped (`-ldflags=-s -w`) for the host, hermetically.
+fn build_go(dir: &std::path::Path, out_bin: &std::path::Path) -> bool {
+    if std::fs::create_dir_all(dir).is_err() {
+        return false;
+    }
+    if std::fs::write(dir.join("main.go"), MAIN_GO).is_err() {
+        return false;
+    }
+    Command::new("go")
+        .args(["build", "-ldflags=-s -w", "-o"])
+        .arg(out_bin)
+        .arg("main.go")
+        .current_dir(dir)
+        .env("GOCACHE", dir.join(".gocache"))
+        .env("GOPATH", dir.join(".gopath"))
+        .env("CGO_ENABLED", "0")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .map(|s| s.success())
+        .unwrap_or(false)
+}
+
 /// The virtual address of [`WITNESS`] inside the built binary's `.rodata`.
 fn witness_vma(bin: &std::path::Path) -> Option<u64> {
     let bytes = std::fs::read(bin).ok()?;
@@ -160,3 +211,41 @@ fn rust_str_records_segment_the_glued_nul_run() {
 
     let _ = std::fs::remove_dir_all(&dir);
 }
+
+/// The Go arm: the same record shape, slots in the Go data image, under the
+/// same option. A stripped Go build's table headers state the literal's exact
+/// 21 bytes; `off` leaves the blob to the NUL scan, which cannot.
+#[test]
+fn go_string_records_bound_the_blob_literal() {
+    if !go_available() {
+        eprintln!("verify_fatstr_records: skipping Go arm (`go` not on PATH)");
+        return;
+    }
+    let dir = std::env::temp_dir().join(format!("kuna_fatstr_go_{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    let out_bin = dir.join("go_fatstr");
+    if !build_go(&dir, &out_bin) {
+        let _ = std::fs::remove_dir_all(&dir);
+        panic!("installed Go compiler failed to build the fixture");
+    }
+    let bin = out_bin.to_str().expect("utf8 path");
+    let vma = witness_vma(&out_bin).expect("the witness literal is in .rodata");
+
+    let bounded = committed_extent(bin, vma, None)
+        .expect("fatstr auto commits a bounded symbol at the literal");
+    assert_eq!(
+        bounded,
+        WITNESS.len() as i32,
+        "the Go header-stated extent is the literal's own 21 bytes"
+    );
+
+    match committed_extent(bin, vma, Some("off")) {
+        Some(glued) => assert!(
+            glued != WITNESS.len() as i32,
+            "with fatstr off nothing states the literal's exact extent"
+        ),
+        None => {}
+    }
+
+    let _ = std::fs::remove_dir_all(&dir);
+}