From b1ae9a02f09b0d86042b4f992a4cafc4a1c1b2a2 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 9 Jul 2026 08:21:58 +0800 Subject: [PATCH 1/4] ssa,cl,build: release per-compile memory in in-process drivers The golden test harness (cltest, llgen) compiles ~85 packages inside one process, and each compile's memory lived until process exit, so cl.test RSS climbed monotonically from 140MB to over 2GB. Two retainers: - The runtime-caller tracking memoization (runtimeCallerBaseCache / runtimeCallerExtendedCache) is keyed by *ssa.Package with *ssa.Function values, pinning every compiled package's go/types and go/ssa graphs forever. A heap profile at the end of the Testrt+ Testdata suites showed 1.1GB of retained front-end data (216MB alone in go/types recordTypeAndValue). Clear both maps when build.Do returns. - No LLVM Context/TargetMachine/TargetData was ever disposed, so each compile's C++-side memory also accumulated. Add Program.Dispose and call it when build.Do returns (except ModeGen, whose callers read LPkg.String() afterwards and dispose via llgen.GenFrom). Testrt+Testdata trajectory on darwin/arm64 (5s RSS samples, cold llgo cache): before 140MB -> 2051MB peak; after 127MB -> 537MB peak (-74%). Single-shot llgo CLI behavior is unchanged. Co-Authored-By: Claude Fable 5 --- cl/instr.go | 10 ++++++++++ internal/build/build.go | 13 +++++++++++++ internal/llgen/llgenf.go | 7 ++++++- ssa/package.go | 18 ++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/cl/instr.go b/cl/instr.go index 994b438c18..34e2d0089d 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -969,6 +969,16 @@ var ( runtimeCallerExtendedCache sync.Map // *ssa.Package -> map[*ssa.Function]bool ) +// ClearRuntimeCallerCaches drops the caller-tracking memoization. The maps +// are keyed by *ssa.Package pointers and their values hold *ssa.Function, +// which pins every compiled package's go/types and go/ssa graphs for the +// life of the process; in-process drivers that compile many packages +// sequentially (the golden test harness) call this between compiles. +func ClearRuntimeCallerCaches() { + runtimeCallerBaseCache.Clear() + runtimeCallerExtendedCache.Clear() +} + func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { if fn == nil || fn.Parent() != nil { return false diff --git a/internal/build/build.go b/internal/build/build.go index 00e41ce4c3..be3b0570f8 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -322,6 +322,19 @@ func Do(args []string, conf *Config) ([]Package, error) { } prog := llssa.NewProgram(target) + // The caller-tracking memoization pins each compiled package's + // front-end graphs (go/types, go/ssa) via *ssa.Package/*ssa.Function + // keys; release them with the compile. + defer cl.ClearRuntimeCallerCaches() + if conf.Mode != ModeGen { + // ModeGen callers (llgen and the golden suites) read LPkg.String() + // after Do returns and dispose the program themselves; every other + // mode's outputs are files or a spawned process, so the compile's + // LLVM context can be released when Do finishes. In-process + // drivers that build many packages per process (the cltest run + // harness) otherwise accumulate every compile's C++-side memory. + defer prog.Dispose() + } prog.EnableGoGlobalDCE(conf.goGlobalDCEEnabled()) if conf.PthreadStackSize > 0 { prog.SetPthreadStackSize(uint64(conf.PthreadStackSize)) diff --git a/internal/llgen/llgenf.go b/internal/llgen/llgenf.go index 4ebb8f1d60..fa51fc45c1 100644 --- a/internal/llgen/llgenf.go +++ b/internal/llgen/llgenf.go @@ -28,7 +28,12 @@ import ( func GenFrom(fileOrPkg string) string { pkg, err := genFrom(fileOrPkg, 0) check(err) - return pkg.LPkg.String() + out := pkg.LPkg.String() + // Release the compile's LLVM context: golden suites call GenFrom for + // every test dir inside one process, and undisposed contexts + // accumulate C++-side memory for the whole run. + pkg.LPkg.Prog.Dispose() + return out } func genFrom(pkgPath string, abiMode build.AbiMode) (build.Package, error) { diff --git a/ssa/package.go b/ssa/package.go index 1ceda02dc9..5b0e9dd411 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -231,6 +231,7 @@ type aProgram struct { abi abi.Builder is32Bits bool + disposed bool enableGoGlobalDCE bool pthreadStackSize uint64 @@ -267,6 +268,23 @@ func is32Bits(arch string) bool { return false } +// Dispose releases the LLVM resources owned by the program: the context +// (and with it every module built in it), the target machine and the +// target data. The Program and everything created from it must not be +// used afterwards. In-process drivers that compile many packages +// sequentially (llgen goldens, the cltest run harness) call this between +// compiles; without it each compile's C++-side memory lives until the +// process exits. +func (p Program) Dispose() { + if p == nil || p.disposed { + return + } + p.disposed = true + p.tm.Dispose() + p.td.Dispose() + p.ctx.Dispose() +} + // NewProgram creates a new program. func NewProgram(target *Target) Program { if target == nil { From c0feb6676758eb6f700c0e69ddb1e0a4ff2122d4 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 9 Jul 2026 08:48:48 +0800 Subject: [PATCH 2/4] internal/cabi: dispose LLVM programs and contexts in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cabi tests drive build.Do in ModeGen (whose Program the caller must dispose) in arch x mode x file loops — hundreds of compiles per process — and also parse reference .ll files into contexts that were never freed. cabi.test peaked at 2.3GB in the CI-suite co-residency profile. Dispose the program (and the parsed-IR module/context) after each check: peak RSS drops 1356MB -> 229MB on darwin/arm64, runtime unchanged. Co-Authored-By: Claude Fable 5 --- internal/cabi/cabi_patch_test.go | 3 +++ internal/cabi/cabi_test.go | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/cabi/cabi_patch_test.go b/internal/cabi/cabi_patch_test.go index 1fb8d03ff7..d2378604cf 100644 --- a/internal/cabi/cabi_patch_test.go +++ b/internal/cabi/cabi_patch_test.go @@ -26,6 +26,7 @@ func TestDevLTOGlobalDCETargetArchAndNewTransformerArchSelection(t *testing.T) { llvm.InitializeAllTargetInfos() prog := llssa.NewProgram(nil) + defer prog.Dispose() tests := []struct { target string abi string @@ -126,6 +127,7 @@ func TestRuntimeHeaderWrapAndTypeInfo(t *testing.T) { llvm.InitializeAllTargetInfos() prog := llssa.NewProgram(nil) + defer prog.Dispose() tr := NewTransformer(prog, "", "", ModeAllFunc, false) ctx := llvm.NewContext() @@ -182,6 +184,7 @@ attributes #0 = { "llgo.reflect.methodbyname"="value" } defer mod.Dispose() prog := llssa.NewProgram(nil) + defer prog.Dispose() tr := NewTransformer(prog, "amd64-unknown-linux-gnu", "", ModeAllFunc, true) tr.TransformModule("test", mod) diff --git a/internal/cabi/cabi_test.go b/internal/cabi/cabi_test.go index ab9616bd82..cae8cdf68c 100644 --- a/internal/cabi/cabi_test.go +++ b/internal/cabi/cabi_test.go @@ -75,10 +75,11 @@ func TestBuild(t *testing.T) { for _, mode := range modes { for _, arch := range archs { conf, _ := buildConf(mode, arch) - _, err := build.Do([]string{"./_testdata/demo/demo.go"}, conf) + pkgs, err := build.Do([]string{"./_testdata/demo/demo.go"}, conf) if err != nil { t.Fatalf("build error: %v-%v %v", arch, mode, err) } + pkgs[0].LPkg.Prog.Dispose() } } } @@ -130,6 +131,9 @@ func testArch(t *testing.T, arch string, archDir string, files []string) { tr.TransformModule(file, pkg.Module()) } testModule(t, context{arch: arch, file: file}, pkg.Prog.TargetData(), pkg.Module(), m) + m.Dispose() + ctx.Dispose() + pkg.Prog.Dispose() } } @@ -353,6 +357,7 @@ entry: } prog := pkgs[0].LPkg.Prog + defer prog.Dispose() // Apply CABI transformation with optimize=true // This tests the code path that previously had the memcpy bug @@ -426,6 +431,7 @@ entry: if err != nil { t.Fatalf("Failed to build demo: %v", err) } + defer pkgs[0].LPkg.Prog.Dispose() tr := cabi.NewTransformer(pkgs[0].LPkg.Prog, "", "", cabi.ModeAllFunc, true) tr.TransformModule("test", mod) @@ -492,6 +498,7 @@ entry: t.Fatalf("Failed to build demo: %v", err) } prog := pkgs[0].LPkg.Prog + defer prog.Dispose() tr := cabi.NewTransformer(prog, "", "", cabi.ModeCFunc, true) tr.TransformModule("test", mod) @@ -550,6 +557,7 @@ entry: t.Fatalf("Failed to build demo: %v", err) } prog := pkgs[0].LPkg.Prog + defer prog.Dispose() tr := cabi.NewTransformer(prog, "", "", cabi.ModeAllFunc, true) tr.SetSkipFuncs([]string{"pkg.asm"}) @@ -620,6 +628,7 @@ entry: t.Fatalf("Failed to build demo: %v", err) } prog := pkgs[0].LPkg.Prog + defer prog.Dispose() tr := cabi.NewTransformer(prog, "", "", cabi.ModeAllFunc, true) tr.SetSkipFuncs([]string{"github.com/goplus/llgo/runtime/internal/runtime.Typedslicecopy"}) @@ -685,6 +694,7 @@ entry: t.Fatalf("Failed to build demo: %v", err) } prog := pkgs[0].LPkg.Prog + defer prog.Dispose() tr := cabi.NewTransformer(prog, "", "", cabi.ModeAllFunc, true) tr.TransformModule("test", mod) @@ -750,6 +760,7 @@ entry: t.Fatalf("Failed to build demo: %v", err) } prog := pkgs[0].LPkg.Prog + defer prog.Dispose() tr := cabi.NewTransformer(prog, "", "", cabi.ModeAllFunc, true) tr.TransformModule("test", mod) From c9bd8b8a985818eb912805dcb71412c979ea8674 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 9 Jul 2026 08:55:07 +0800 Subject: [PATCH 3/4] ssa: unit-test Program.Dispose nil-safety and idempotence Co-Authored-By: Claude Fable 5 --- ssa/dispose_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 ssa/dispose_test.go diff --git a/ssa/dispose_test.go b/ssa/dispose_test.go new file mode 100644 index 0000000000..1e9b9a94fd --- /dev/null +++ b/ssa/dispose_test.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import "testing" + +func TestProgramDispose(t *testing.T) { + var nilProg Program + nilProg.Dispose() // nil-safe + + prog := NewProgram(nil) + prog.NewPackage("foo", "foo") + prog.Dispose() + if !prog.disposed { + t.Fatal("Dispose should mark the program disposed") + } + prog.Dispose() // idempotent +} From 15215932c6441ccc0efc6b0c14a56f8deaec2941 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 9 Jul 2026 10:22:52 +0800 Subject: [PATCH 4/4] cl,build: compilation-scoped CallerTracking instead of global caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up to the per-Do cache clear: rather than global sync.Maps that build.Do must remember to clear, the caller-tracking memoization is now an exported cl.CallerTracking created once per compilation and passed to NewPackageExWithEmbed for every package of that compilation — the same driver-owned, compilation-scoped flow as cl.Patches (build.Do's context holds both side by side). cl keeps no package-level data state, so nothing can outlive a compile and pin its go/types and go/ssa graphs; ClearRuntimeCallerCaches is gone. NewPackage and NewPackageEx keep their signatures and compile as one-shot compilations (fresh memoization per call); passing nil to NewPackageExWithEmbed does the same. Memory behavior is identical to the cleared-globals version: Testrt+Testdata trajectory 129MB -> 562MB peak with a plateau (was 140MB -> 2051MB before the fix), cl/ssa/build/ cabi suites all pass. Discussed in #2049 (now closed: implemented here). Co-Authored-By: Claude Fable 5 --- cl/caller_frame_test.go | 35 +++++++++-------- cl/caller_tracking_compile_test.go | 58 +++++++++++++++++++++++++++++ cl/compile.go | 23 +++++++++--- cl/embed_with_map_compile_test.go | 2 +- cl/instr.go | 60 ++++++++++++++++-------------- internal/build/build.go | 10 ++--- 6 files changed, 132 insertions(+), 56 deletions(-) create mode 100644 cl/caller_tracking_compile_test.go diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 9cbb5b9843..9879a39585 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -196,25 +196,26 @@ func callWorker(w workerIface) { w.Work() } func workerHot() { var w workerIface = workerImpl{}; callWorker(w) } func plain() {} `) - if !packageUsesRuntimeCaller(ssapkg) { + callerCaches := NewCallerTracking() + if !packageUsesRuntimeCaller(callerCaches, ssapkg) { t.Fatal("package should report runtime caller usage") } - if !fnUsesRuntimeCaller(ssapkg.Func("direct")) { + if !fnUsesRuntimeCaller(callerCaches, ssapkg.Func("direct")) { t.Fatal("direct runtime.Caller use should be detected") } - if !fnUsesRuntimeCaller(ssapkg.Func("indirect")) { + if !fnUsesRuntimeCaller(callerCaches, ssapkg.Func("indirect")) { t.Fatal("transitive runtime.Caller use should be detected") } - if !fnUsesRuntimeCaller(ssapkg.Func("stack")) { + if !fnUsesRuntimeCaller(callerCaches, ssapkg.Func("stack")) { t.Fatal("runtime/debug.Stack use should be detected") } - if !fnUsesRuntimeCaller(ssapkg.Func("anonOnly")) { + if !fnUsesRuntimeCaller(callerCaches, ssapkg.Func("anonOnly")) { t.Fatal("runtime caller use in anonymous functions should be detected") } - if fnUsesRuntimeCaller(ssapkg.Func("plain")) { + if fnUsesRuntimeCaller(callerCaches, ssapkg.Func("plain")) { t.Fatal("plain function should not report runtime caller usage") } - runtimeCallerFuncs := runtimeCallerFuncSet(ssapkg) + runtimeCallerFuncs := runtimeCallerFuncSet(callerCaches, ssapkg) for _, name := range []string{"dynamic", "dynamicCaller", "interfaceDispatch", "interfaceCaller", "closureLayer", "closureCaller"} { if !runtimeCallerFuncs[ssapkg.Func(name)] { t.Fatalf("%s should be tracked because dynamic calls may reach runtime stack APIs", name) @@ -266,13 +267,14 @@ func FuncForPC(pc uintptr) uintptr { return 0 } } func TestRuntimeCallerAnalysisEdgeCases(t *testing.T) { - if fnUsesRuntimeCaller(nil) { + callerCaches := NewCallerTracking() + if fnUsesRuntimeCaller(callerCaches, nil) { t.Fatal("nil function should not use runtime caller metadata") } - if fnUsesRuntimeCaller(&gossa.Function{}) { + if fnUsesRuntimeCaller(callerCaches, &gossa.Function{}) { t.Fatal("function without a package should not use runtime caller metadata") } - if runtimeCallerFuncSet(nil) != nil { + if runtimeCallerFuncSet(callerCaches, nil) != nil { t.Fatal("nil package should have no runtime caller set") } if fnHasDirectRuntimeCaller(nil) { @@ -381,7 +383,7 @@ type T struct{} func (T) Call() { runtime.Caller(0) } var _ = T{} `) - methodOnlySet := runtimeCallerFuncSet(methodOnlyPkg) + methodOnlySet := runtimeCallerFuncSet(NewCallerTracking(), methodOnlyPkg) if methodOnlySet == nil { t.Fatal("a method calling runtime.Caller must be tracked (slog.(*Logger).Info escaped exactly this way)") } @@ -445,7 +447,7 @@ func f() { runtime.Caller(0) } fn: fn, goFn: goFn, trackCallerFrames: tt.track, - runtimeCallerFuncs: runtimeCallerFuncSet(ssapkg), + runtimeCallerFuncs: runtimeCallerFuncSet(NewCallerTracking(), ssapkg), } if got := ctx.shouldTrackCallerFrames(); got != tt.want { t.Fatalf("shouldTrackCallerFrames() = %v, want %v", got, tt.want) @@ -650,7 +652,7 @@ func top() { goFn: ssapkg.Func("top"), fset: token.NewFileSet(), trackCallerFrames: true, - runtimeCallerFuncs: runtimeCallerFuncSet(ssapkg), + runtimeCallerFuncs: runtimeCallerFuncSet(NewCallerTracking(), ssapkg), } var b llssa.Builder ctx.pushCallerLocationFrame(b, nil) @@ -940,10 +942,11 @@ func Logs() bool { return dep.Where() } func Plain() int { return dep.Quiet() } `) - if !runtimeCallerBaseSet(depSSA)[depSSA.Func("Where")] { + crossCaches := NewCallerTracking() + if !runtimeCallerBaseSet(crossCaches, depSSA)[depSSA.Func("Where")] { t.Fatal("dep.Where must be in its own package's base set") } - set := runtimeCallerFuncSet(rootSSA) + set := runtimeCallerFuncSet(crossCaches, rootSSA) if !set[rootSSA.Func("Logs")] { t.Fatal("caller of a pc-consuming function must be tracked (criterion 2)") } @@ -966,7 +969,7 @@ func pinned() {} func helper() {} `) - set := runtimeCallerFuncSet(ssapkg) + set := runtimeCallerFuncSet(NewCallerTracking(), ssapkg) for _, name := range []string{"main", "init", "pinned"} { if !set[ssapkg.Func(name)] { t.Fatalf("%s must be pinned in the tracking set", name) diff --git a/cl/caller_tracking_compile_test.go b/cl/caller_tracking_compile_test.go new file mode 100644 index 0000000000..4bcf90dc0c --- /dev/null +++ b/cl/caller_tracking_compile_test.go @@ -0,0 +1,58 @@ +//go:build !llgo +// +build !llgo + +package cl_test + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "runtime" + "testing" + + "github.com/goplus/gogen/packages" + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/ssa/ssatest" + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" +) + +// TestNewPackageExWithEmbedSharedTracking compiles with a caller-provided +// CallerTracking, the multi-package driver pattern where one instance is +// shared across every package of a compilation (like patches). +func TestNewPackageExWithEmbedSharedTracking(t *testing.T) { + src := `package foo + +import "runtime" + +func Where() string { + _, file, _, _ := runtime.Caller(0) + return file +} +` + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "foo.go", src, parser.ParseComments) + if err != nil { + t.Fatalf("ParseFile failed: %v", err) + } + files := []*ast.File{f} + + imp := packages.NewImporter(fset) + mode := ssa.SanityCheckFunctions | ssa.InstantiateGenerics + fooPkg, _, err := ssautil.BuildPackage(&types.Config{Importer: imp}, fset, types.NewPackage(f.Name.Name, f.Name.Name), files, mode) + if err != nil { + t.Fatalf("BuildPackage failed: %v", err) + } + + prog := ssatest.NewProgramEx(t, nil, imp) + prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) + ct := cl.NewCallerTracking() + ret, _, err := cl.NewPackageExWithEmbed(prog, ct, nil, nil, fooPkg, files, nil) + if err != nil { + t.Fatalf("NewPackageExWithEmbed failed: %v", err) + } + if ret.String() == "" { + t.Fatal("empty IR") + } +} diff --git a/cl/compile.go b/cl/compile.go index a4a7fc2aae..b029b45a51 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -1881,6 +1881,10 @@ func NewPackage(prog llssa.Program, pkg *ssa.Package, files []*ast.File) (ret ll return } +// NewPackageEx and NewPackage compile as a one-shot compilation: each +// call gets fresh caller-tracking memoization. Multi-package drivers +// use NewPackageExWithEmbed with a shared CallerTracking instead. + // NewPackageEx compiles a Go package to LLVM IR package. // // Parameters: @@ -1893,17 +1897,21 @@ func NewPackage(prog llssa.Program, pkg *ssa.Package, files []*ast.File) (ret ll // The rewrites map uses short variable names (without package qualifier) and // only affects string-typed globals defined in the current package. func NewPackageEx(prog llssa.Program, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File) (ret llssa.Package, externs []string, err error) { - return newPackageEx(prog, patches, rewrites, pkg, files, nil) + return newPackageEx(prog, nil, patches, rewrites, pkg, files, nil) } // NewPackageExWithEmbed compiles a package using pre-loaded go:embed metadata. // // This avoids re-scanning directives when the caller already loaded them. -func NewPackageExWithEmbed(prog llssa.Program, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap goembed.VarMap) (ret llssa.Package, externs []string, err error) { - return newPackageEx(prog, patches, rewrites, pkg, files, &embedMap) +// ct carries the compilation-scoped caller-tracking memoization; drivers +// compiling multiple packages pass the same instance for every package +// of one compilation (like patches). nil means one-shot: a fresh +// instance is created for this call. +func NewPackageExWithEmbed(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap goembed.VarMap) (ret llssa.Package, externs []string, err error) { + return newPackageEx(prog, ct, patches, rewrites, pkg, files, &embedMap) } -func newPackageEx(prog llssa.Program, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap *goembed.VarMap) (ret llssa.Package, externs []string, err error) { +func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewrites map[string]string, pkg *ssa.Package, files []*ast.File, embedMap *goembed.VarMap) (ret llssa.Package, externs []string, err error) { pkgProg := pkg.Prog pkgTypes := pkg.Pkg oldTypes := pkgTypes @@ -1922,6 +1930,9 @@ func newPackageEx(prog llssa.Program, patches Patches, rewrites map[string]strin ret.InitDebug(pkgName, pkgPath, pkgProg.Fset) } + if ct == nil { + ct = NewCallerTracking() + } ctx := &context{ prog: prog, pkg: ret, @@ -1941,8 +1952,8 @@ func newPackageEx(prog llssa.Program, patches Patches, rewrites map[string]strin cgoSymbols: make([]string, 0, 128), rewrites: rewrites, - trackCallerFrames: filesUseRuntimeCaller(files) || packageUsesRuntimeCaller(pkg), - runtimeCallerFuncs: runtimeCallerFuncSet(pkg), + trackCallerFrames: filesUseRuntimeCaller(files) || packageUsesRuntimeCaller(ct, pkg), + runtimeCallerFuncs: runtimeCallerFuncSet(ct, pkg), } if embedMap != nil { ctx.embedMap = *embedMap diff --git a/cl/embed_with_map_compile_test.go b/cl/embed_with_map_compile_test.go index 069be194d1..48a161e1e7 100644 --- a/cl/embed_with_map_compile_test.go +++ b/cl/embed_with_map_compile_test.go @@ -57,7 +57,7 @@ var files embed.FS prog := ssatest.NewProgramEx(t, nil, imp) prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) - ret, _, err := cl.NewPackageExWithEmbed(prog, nil, nil, fooPkg, files, embedMap) + ret, _, err := cl.NewPackageExWithEmbed(prog, nil, nil, nil, fooPkg, files, embedMap) if err != nil { t.Fatalf("NewPackageExWithEmbed failed: %v", err) } diff --git a/cl/instr.go b/cl/instr.go index 34e2d0089d..60e83145e7 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -26,7 +26,6 @@ import ( "os" "regexp" "strings" - "sync" "golang.org/x/tools/go/ssa" @@ -880,18 +879,18 @@ func canTrackCallerFramesForPackage(pkgPath string) bool { !strings.HasPrefix(pkgPath, "github.com/goplus/llgo/runtime/internal/") } -func packageUsesRuntimeCaller(pkg *ssa.Package) bool { - return len(runtimeCallerFuncSet(pkg)) != 0 +func packageUsesRuntimeCaller(c *CallerTracking, pkg *ssa.Package) bool { + return len(runtimeCallerFuncSet(c, pkg)) != 0 } -func fnUsesRuntimeCaller(fn *ssa.Function) bool { +func fnUsesRuntimeCaller(c *CallerTracking, fn *ssa.Function) bool { if fn == nil { return false } if fn.Pkg == nil { return fnHasDirectRuntimeCaller(fn) } - return runtimeCallerFuncSet(fn.Pkg)[fn] + return runtimeCallerFuncSet(c, fn.Pkg)[fn] } // runtimeCallerFuncSet is the per-package tracking set: functions that @@ -910,15 +909,14 @@ func fnUsesRuntimeCaller(fn *ssa.Function) bool { // (criterion 1 alone), so tracking extends exactly one call level past a // pc-consuming package and does not cascade through arbitrary wrapper // layers; multi-package wrapper chains remain the P4 inline-tree's job. -func runtimeCallerFuncSet(pkg *ssa.Package) map[*ssa.Function]bool { +func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function]bool { if pkg == nil { return nil } - if v, ok := runtimeCallerExtendedCache.Load(pkg); ok { - set, _ := v.(map[*ssa.Function]bool) + if set, ok := c.extended[pkg]; ok { return set } - base := runtimeCallerBaseSet(pkg) + base := runtimeCallerBaseSet(c, pkg) out := make(map[*ssa.Function]bool, len(base)) for fn := range base { out[fn] = true @@ -952,7 +950,7 @@ func runtimeCallerFuncSet(pkg *ssa.Package) map[*ssa.Function]bool { if !canTrackCallerFramesForPackage(callee.Pkg.Pkg.Path()) { return } - if runtimeCallerBaseSet(callee.Pkg)[callee] { + if runtimeCallerBaseSet(c, callee.Pkg)[callee] { out[fn] = true } }) @@ -960,23 +958,32 @@ func runtimeCallerFuncSet(pkg *ssa.Package) map[*ssa.Function]bool { if len(out) == 0 { out = nil } - runtimeCallerExtendedCache.Store(pkg, out) + c.extended[pkg] = out return out } -var ( - runtimeCallerBaseCache sync.Map // *ssa.Package -> map[*ssa.Function]bool - runtimeCallerExtendedCache sync.Map // *ssa.Package -> map[*ssa.Function]bool -) +// CallerTracking memoizes the per-package caller-tracking sets for one +// compilation. Like Patches, it is compilation-scoped state owned by the +// driver: create one per compilation and pass it to every +// NewPackageExWithEmbed call of that compilation, so cross-package +// queries (criterion 2 below) hit the memoization. It must not outlive +// the compilation — the maps are keyed by *ssa.Package with +// *ssa.Function values, so anything longer-lived would pin every +// compiled package's go/types and go/ssa graphs. Plain maps are enough: +// packages of one compilation are compiled sequentially (the LLVM +// context is not thread-safe). +type CallerTracking struct { + base map[*ssa.Package]map[*ssa.Function]bool + extended map[*ssa.Package]map[*ssa.Function]bool +} -// ClearRuntimeCallerCaches drops the caller-tracking memoization. The maps -// are keyed by *ssa.Package pointers and their values hold *ssa.Function, -// which pins every compiled package's go/types and go/ssa graphs for the -// life of the process; in-process drivers that compile many packages -// sequentially (the golden test harness) call this between compiles. -func ClearRuntimeCallerCaches() { - runtimeCallerBaseCache.Clear() - runtimeCallerExtendedCache.Clear() +// NewCallerTracking creates the caller-tracking memoization for one +// compilation. +func NewCallerTracking() *CallerTracking { + return &CallerTracking{ + base: make(map[*ssa.Package]map[*ssa.Function]bool), + extended: make(map[*ssa.Package]map[*ssa.Function]bool), + } } func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { @@ -990,16 +997,15 @@ func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { return name == "main" && pkg.Pkg != nil && pkg.Pkg.Name() == "main" } -func runtimeCallerBaseSet(pkg *ssa.Package) map[*ssa.Function]bool { +func runtimeCallerBaseSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function]bool { if pkg == nil { return nil } - if v, ok := runtimeCallerBaseCache.Load(pkg); ok { - set, _ := v.(map[*ssa.Function]bool) + if set, ok := c.base[pkg]; ok { return set } set := computeRuntimeCallerBaseSet(pkg) - runtimeCallerBaseCache.Store(pkg, set) + c.base[pkg] = set return set } diff --git a/internal/build/build.go b/internal/build/build.go index be3b0570f8..e561ee1c2e 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -322,10 +322,6 @@ func Do(args []string, conf *Config) ([]Package, error) { } prog := llssa.NewProgram(target) - // The caller-tracking memoization pins each compiled package's - // front-end graphs (go/types, go/ssa) via *ssa.Package/*ssa.Function - // keys; release them with the compile. - defer cl.ClearRuntimeCallerCaches() if conf.Mode != ModeGen { // ModeGen callers (llgen and the golden suites) read LPkg.String() // after Do returns and dispose the program themselves; every other @@ -445,7 +441,8 @@ func Do(args []string, conf *Config) ([]Package, error) { output := conf.OutFile != "" ctx := &context{env: env, conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, - patches: patches, built: make(map[string]none), initial: initial, mode: mode, + patches: patches, callerTracking: cl.NewCallerTracking(), + built: make(map[string]none), initial: initial, mode: mode, fingerprinting: make(map[string]bool), pkgs: map[*packages.Package]Package{}, pkgByID: map[string]Package{}, @@ -622,6 +619,7 @@ type context struct { prog llssa.Program dedup packages.Deduper patches cl.Patches + callerTracking *cl.CallerTracking built map[string]none fingerprinting map[string]bool initial []*packages.Package @@ -1383,7 +1381,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { return fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) } - ret, externs, err := cl.NewPackageExWithEmbed(ctx.prog, ctx.patches, aPkg.rewriteVars, aPkg.SSA, syntax, embedMap) + ret, externs, err := cl.NewPackageExWithEmbed(ctx.prog, ctx.callerTracking, ctx.patches, aPkg.rewriteVars, aPkg.SSA, syntax, embedMap) check(err) aPkg.LPkg = ret