From 958ac93b023c1feb8f1892a94907c516eecec22b Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 01:26:30 +0800 Subject: [PATCH 01/23] build: prepare bounded parallel compilation --- internal/build/build.go | 285 ++++++++++++++++++---------- internal/build/collect.go | 35 ++-- internal/build/dependencies.go | 52 +++++ internal/build/dependencies_test.go | 46 +++++ internal/build/fingerprint_test.go | 7 +- internal/build/module_hook_test.go | 1 + internal/goflags/flagfile.go | 1 + internal/goflags/gobuild.go | 33 +++- internal/goflags/gobuild_test.go | 25 +++ 9 files changed, 364 insertions(+), 121 deletions(-) create mode 100644 internal/build/dependencies.go create mode 100644 internal/build/dependencies_test.go diff --git a/internal/build/build.go b/internal/build/build.go index bbabf4b969..870c570d3d 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -180,6 +180,9 @@ type Config struct { // default. PCLNModeSet bool AllowNoBody bool // allow declarations without bodies, as go tool compile does + // Parallel is the maximum number of LLGo's internally parallel build tasks. + // Zero uses GOMAXPROCS, matching the Go command's default -p behavior. + Parallel int // PthreadStackSize sets a custom stack size, in bytes, for pthread-backed // goroutines. A zero value keeps the platform pthread default. @@ -278,7 +281,19 @@ const ( loadSyntax = loadTypes | packages.NeedSyntax | packages.NeedTypesInfo ) +var llssaInitOnce sync.Once + +func (c *Config) parallelism() int { + if c != nil && c.Parallel > 0 { + return c.Parallel + } + return runtime.GOMAXPROCS(0) +} + func Do(args []string, conf *Config) ([]Package, error) { + if conf.Parallel < 0 { + return nil, fmt.Errorf("parallelism must not be negative: %d", conf.Parallel) + } if conf.Goos == "" { conf.Goos = runtime.GOOS } @@ -369,7 +384,9 @@ func Do(args []string, conf *Config) ([]Package, error) { cl.EnableDebug(emitDebugInfo) cl.EnableDbgSyms(emitDebugInfo) cl.EnableTrace(IsTraceEnabled()) - llssa.Initialize(llssa.InitAll) + llssaInitOnce.Do(func() { + llssa.Initialize(llssa.InitAll) + }) target := &llssa.Target{ GOOS: conf.Goos, @@ -494,7 +511,6 @@ func Do(args []string, conf *Config) ([]Package, error) { } progSSA := ssa.NewProgram(initial[0].Fset, buildMode) patches := make(cl.Patches, len(altPkgPaths)) - altSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) env := llvm.New("") os.Setenv("PATH", env.BinDir()+":"+os.Getenv("PATH")) // TODO(xsw): check windows @@ -511,20 +527,24 @@ func Do(args []string, conf *Config) ([]Package, error) { buildConf: conf, crossCompile: export, cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + sfilesCache: make(map[string][]string), } defer ctx.closePackageMetas() + ctx.initializePackageBuildState() // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) addGlobalString(conf, "runtime.buildVersion="+runtime.Version(), nil) - pkgs, err := buildSSAPkgs(ctx, initial, verbose) + altEntries := registerAltSSAPkgs(progSSA, patches, altPkgs[1:], conf, verbose) + pkgs, pkgEntries, err := registerSSAPkgs(ctx, initial, verbose) if err != nil { return nil, err } - depPkgs, err := buildSSAPkgs(ctx, altPkgs, verbose) + depPkgs, depEntries, err := registerSSAPkgs(ctx, altPkgs, verbose) if err != nil { return nil, err } + buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) @@ -753,8 +773,9 @@ type context struct { testFail bool // Cache related fields - cacheManager *cacheManager - llvmVersion string + cacheManager *cacheManager + llvmVersion string + llvmVersionReady bool // go list derived file lists (SFiles, etc.) sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths @@ -841,8 +862,6 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { - built := ctx.built - // Split packages into runtime tree vs others so we can defer runtime build. var runtimePkgs []*aPackage var normalPkgs []*aPackage @@ -856,91 +875,20 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er var needRuntime, needPyInit bool - buildOne := func(aPkg *aPackage) error { - pkg := aPkg.Package - if _, ok := built[pkg.ID]; ok { - // Already built, skip but keep ExportFile for linking - return nil - } - built[pkg.ID] = none{} - - switch kind, param := cl.PkgKindOf(pkg.Types); kind { - case cl.PkgDeclOnly: - pkg.ExportFile = "" - case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule: - if len(pkg.GoFiles) > 0 { - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } else { - pkg.ExportFile = "" - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) - } - } - default: - if err := ctx.collectFingerprint(aPkg); err != nil { - return err - } - ctx.tryLoadFromCache(aPkg) - if verbose { - if aPkg.CacheHit { - fmt.Fprintf(os.Stderr, "CACHE HIT: %s\n", pkg.PkgPath) - } else { - fmt.Fprintf(os.Stderr, "CACHE MISS: %s\n", pkg.PkgPath) - } - } - if err := buildPkg(ctx, aPkg, verbose); err != nil { - return err - } - aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) - needRuntime = needRuntime || aPkg.NeedRt - needPyInit = needPyInit || aPkg.NeedPyInit - if !aPkg.CacheHit { - if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return err - } - if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) - } - } - } - return nil - } - // Build non-runtime packages first, so we know whether runtime is actually needed. for _, p := range normalPkgs { - if err := buildOne(p); err != nil { + result, err := buildOnePackage(ctx, p, verbose) + if err != nil { return nil, err } + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit } // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { for _, p := range runtimePkgs { - if err := buildOne(p); err != nil { + if _, err := buildOnePackage(ctx, p, verbose); err != nil { return nil, err } } @@ -949,6 +897,69 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return pkgs, nil } +type packageBuildResult struct { + needRuntime bool + needPyInit bool +} + +// buildOnePackage is the serial package pipeline. The stages are separated so +// a future scheduler can parallelize only stages with isolated package state. +func buildOnePackage(ctx *context, aPkg *aPackage, verbose bool) (packageBuildResult, error) { + pkg := aPkg.Package + if _, ok := ctx.built[pkg.ID]; ok { + return packageBuildResult{}, nil + } + ctx.built[pkg.ID] = none{} + + kind, param := cl.PkgKindOf(pkg.Types) + if kind == cl.PkgDeclOnly { + pkg.ExportFile = "" + return packageBuildResult{}, nil + } + if (kind == cl.PkgLinkIR || kind == cl.PkgLinkExtern || kind == cl.PkgPyModule) && len(pkg.GoFiles) == 0 { + pkg.ExportFile = "" + if kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, param) + } + return packageBuildResult{}, nil + } + + if err := ctx.collectFingerprint(aPkg); err != nil { + return packageBuildResult{}, err + } + ctx.tryLoadFromCache(aPkg) + if verbose { + status := "MISS" + if aPkg.CacheHit { + status = "HIT" + } + fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) + } + if err := buildPkg(ctx, aPkg, verbose); err != nil { + return packageBuildResult{}, err + } + + result := packageBuildResult{} + if kind != cl.PkgLinkIR && kind != cl.PkgLinkExtern && kind != cl.PkgPyModule { + aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) + result.needRuntime = aPkg.NeedRt + result.needPyInit = aPkg.NeedPyInit + } + if aPkg.CacheHit { + return result, nil + } + if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { + return packageBuildResult{}, err + } + if kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, param) + } + if err := ctx.saveToCache(aPkg); err != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) + } + return result, nil +} + func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { // need to be linked with external library // format: ';' separated alternative link methods. e.g. @@ -1598,6 +1609,16 @@ func is32Bits(goarch string) bool { } func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { + externs, err := preparePackageModule(ctx, aPkg, verbose) + if err != nil || aPkg.CacheHit || aPkg.LPkg == nil { + return err + } + return compilePackageModule(ctx, aPkg, externs, verbose) +} + +// preparePackageModule runs the frontend and creates the package LLVM module. +// This stage remains serial because it updates Program-wide registration state. +func preparePackageModule(ctx *context, aPkg *aPackage, verbose bool) ([]string, error) { pkg := aPkg.Package pkgPath := pkg.PkgPath if debugBuild || verbose { @@ -1607,7 +1628,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { } if llruntime.SkipToBuild(pkgPath) { pkg.ExportFile = "" - return nil + return nil, nil } var syntax = pkg.Syntax if altPkg := aPkg.AltPkg; altPkg != nil { @@ -1625,7 +1646,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { embedMap, err := goembed.LoadDirectives(ctx.conf.Fset, syntax) if err != nil { - return fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) + return nil, fmt.Errorf("load go:embed directives for %s failed: %w", pkgPath, err) } needMeta := !aPkg.CacheHit && ctx.buildConf.packageMetaEnabled() @@ -1642,8 +1663,18 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { // If cache hit, we only needed to register types - skip compilation if aPkg.CacheHit { - return nil + return nil, nil } + return externs, nil +} + +// compilePackageModule applies LLVM transforms and emits package objects. +// It receives a module owned by aPkg and is intentionally kept separate from +// frontend setup so later PRs can give it a worker-local backend context. +func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbose bool) error { + pkg := aPkg.Package + pkgPath := pkg.PkgPath + ret := aPkg.LPkg ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module())) llabi.LowerLargeAggregates(ctx.prog.TargetData(), ret.Module()) @@ -1657,7 +1688,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { mod.SetTarget(ctx.prog.Target().Spec().Triple) pbo := gllvm.NewPassBuilderOptions() defer pbo.Dispose() - if err = gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { + if err := gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { return err } if err := mod.RunPasses(llvmPassPipeline(ctx.buildConf.OptLevel, ctx.buildConf.ltoMode()), ctx.prog.TargetMachine(), pbo); err != nil { @@ -1913,13 +1944,24 @@ func preCollectRuntimeLinknames(prog llssa.Program, pkgs []*packages.Package) { } } -func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) { +type ssaBuildEntry struct { + pkg *ssa.Package + syntax []*ast.File + fixOrder bool +} + +// registerAltSSAPkgs creates the alternate packages and patch table before any +// SSA package is built. Building is deliberately deferred so every package can +// be built under the same bounded worker pool. +func registerAltSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) []ssaBuildEntry { + var entries []ssaBuildEntry packages.Visit(alts, nil, func(p *packages.Package) { if typs := p.Types; typs != nil && !p.IllTyped { if debugBuild || verbose { log.Println("==> BuildSSA", p.ID) } pkgSSA := prog.CreatePackage(typs, p.Syntax, p.TypesInfo, true) + entries = append(entries, ssaBuildEntry{pkg: pkgSSA, syntax: p.Syntax}) if strings.HasPrefix(p.ID, altPkgPathPrefix) { path := p.ID[len(altPkgPathPrefix):] // Even if an alt package exists and is pulled in as a dependency of other @@ -1935,7 +1977,7 @@ func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, } } }) - prog.Build() + return entries } type aPackage struct { @@ -1961,9 +2003,13 @@ type aPackage struct { type Package = *aPackage -func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, error) { +// registerSSAPkgs creates all ordinary SSA packages and returns their build +// entries. It intentionally performs no Package.Build calls: applying patches +// and registering the full package graph must remain serial. +func registerSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, []ssaBuildEntry, error) { prog := ctx.progSSA var all []*aPackage + var entries []ssaBuildEntry var errs []*packages.Package packages.Visit(initial, nil, func(p *packages.Package) { if p.Types != nil && !p.IllTyped { @@ -1973,7 +2019,10 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a return } var altPkg *packages.Cached - var ssaPkg = createSSAPkg(ctx, prog, p, verbose) + ssaPkg, created := createSSAPkg(ctx, prog, p, verbose) + if created { + entries = append(entries, ssaBuildEntry{pkg: ssaPkg, syntax: p.Syntax, fixOrder: true}) + } if ctx.hasAltPkg(pkgPath) { if altPkg = ctx.dedup.Check(altPkgPathPrefix + pkgPath); altPkg == nil { return @@ -2005,9 +2054,49 @@ func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*a } fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg) } - return nil, fmt.Errorf("cannot build SSA for packages") + return nil, nil, fmt.Errorf("cannot build SSA for packages") + } + return all, entries, nil +} + +// buildSSAPkgs builds registered packages with the configured bound, then +// performs ordering repair serially because it mutates package instruction +// slices. go/ssa.Package.Build is documented as safe for concurrent calls. +func buildSSAPkgs(ctx *context, entries []ssaBuildEntry) { + if len(entries) == 0 { + return + } + unique := make([]ssaBuildEntry, 0, len(entries)) + seen := make(map[*ssa.Package]bool, len(entries)) + for _, entry := range entries { + if entry.pkg == nil || seen[entry.pkg] { + continue + } + seen[entry.pkg] = true + unique = append(unique, entry) + } + workers := min(ctx.buildConf.parallelism(), len(unique)) + jobs := make(chan ssaBuildEntry) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for entry := range jobs { + entry.pkg.Build() + } + }() + } + for _, entry := range unique { + jobs <- entry + } + close(jobs) + wg.Wait() + for _, entry := range unique { + if entry.fixOrder { + fixSSAOrder(entry.pkg, entry.syntax) + } } - return all, nil } func formatPackageError(err packages.Error, noColumn bool) string { @@ -2155,7 +2244,7 @@ func applyPatches(ctx *context, p *packages.Package, verbose bool) { } } -func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) *ssa.Package { +func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose bool) (*ssa.Package, bool) { pkgSSA := prog.ImportedPackage(p.ID) if pkgSSA == nil { if debugBuild || verbose { @@ -2163,11 +2252,9 @@ func createSSAPkg(ctx *context, prog *ssa.Program, p *packages.Package, verbose } applyPatches(ctx, p, verbose) pkgSSA = prog.CreatePackage(p.Types, p.Syntax, p.TypesInfo, true) - pkgSSA.Build() // TODO(xsw): build concurrently - // Apply local SSA fixups once when package SSA is first built. - fixSSAOrder(pkgSSA, p.Syntax) + return pkgSSA, true } - return pkgSSA + return pkgSSA, false } /* diff --git a/internal/build/collect.go b/internal/build/collect.go index 72ed06b9f6..7a41e427ae 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -23,7 +23,6 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" "strconv" "strings" @@ -195,21 +194,7 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error // collectDependencyInputs adds dependency fingerprints/versions into manifest. func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) error { - if len(pkg.Imports) == 0 { - return nil - } - - deps := make([]*packages.Package, 0, len(pkg.Imports)) - for _, dep := range pkg.Imports { - if dep == nil || dep.ID == pkg.ID { - continue - } - deps = append(deps, dep) - } - - sort.Slice(deps, func(i, j int) bool { return deps[i].ID < deps[j].ID }) - - for _, dep := range deps { + for _, dep := range effectiveDependencies(pkg) { depEntry, err := c.dependencyFingerprint(dep) if err != nil { return err @@ -220,6 +205,21 @@ func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) err return nil } +// initializePackageBuildState initializes the mutable state used by the +// package pipeline before any scheduler is introduced. This makes ownership +// explicit and avoids lazy first-use writes becoming data races later. +func (c *context) initializePackageBuildState() { + if c.sfilesCache == nil { + c.sfilesCache = make(map[string][]string) + } + if !cacheEnabled() { + return + } + c.cacheManager = newCacheManager() + c.llvmVersion = detectLLVMVersion(c) + c.llvmVersionReady = true +} + func (c *context) dependencyFingerprint(dep *packages.Package) (depEntry, error) { entry := depEntry{ID: dep.ID} if v := moduleVersion(dep.Module); v != "" { @@ -263,10 +263,11 @@ func moduleVersion(mod *gopackages.Module) string { // getLLVMVersion returns the cached LLVM version or detects it. func (c *context) getLLVMVersion() string { - if c.llvmVersion != "" { + if c.llvmVersionReady { return c.llvmVersion } c.llvmVersion = detectLLVMVersion(c) + c.llvmVersionReady = true return c.llvmVersion } diff --git a/internal/build/dependencies.go b/internal/build/dependencies.go new file mode 100644 index 0000000000..a2197623d0 --- /dev/null +++ b/internal/build/dependencies.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + + "github.com/goplus/llgo/internal/packages" +) + +// effectiveDependencies returns the package graph that contributes code to an +// aPackage. An alternate package can add imports beyond the original package, +// so using only Package.Imports would allow stale cache entries after an alt +// dependency changes. +func effectiveDependencies(pkg *aPackage) []*packages.Package { + if pkg == nil || pkg.Package == nil { + return nil + } + deps := make(map[string]*packages.Package) + add := func(imports map[string]*packages.Package) { + for _, dep := range imports { + if dep == nil || dep.ID == pkg.ID || (pkg.AltPkg != nil && dep.ID == pkg.AltPkg.ID) { + continue + } + deps[dep.ID] = dep + } + } + add(pkg.Imports) + if pkg.AltPkg != nil { + add(pkg.AltPkg.Imports) + } + ret := make([]*packages.Package, 0, len(deps)) + for _, dep := range deps { + ret = append(ret, dep) + } + sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) + return ret +} diff --git a/internal/build/dependencies_test.go b/internal/build/dependencies_test.go new file mode 100644 index 0000000000..7f083eb13d --- /dev/null +++ b/internal/build/dependencies_test.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "reflect" + "testing" + + "github.com/goplus/llgo/internal/packages" +) + +func TestEffectiveDependenciesIncludesAlternateImports(t *testing.T) { + base := &packages.Package{ID: "base"} + shared := &packages.Package{ID: "shared"} + altOnly := &packages.Package{ID: "alt-only"} + alt := &packages.Package{ + ID: "alt", + Imports: map[string]*packages.Package{"shared": shared, "alt-only": altOnly}, + } + pkg := &aPackage{ + Package: &packages.Package{ID: "pkg", Imports: map[string]*packages.Package{"base": base, "shared": shared}}, + AltPkg: &packages.Cached{Package: alt}, + } + deps := effectiveDependencies(pkg) + got := make([]string, len(deps)) + for i, dep := range deps { + got[i] = dep.ID + } + if want := []string{"alt-only", "base", "shared"}; !reflect.DeepEqual(got, want) { + t.Fatalf("effectiveDependencies = %v, want %v", got, want) + } +} diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go index 54bba13694..9f765afe21 100644 --- a/internal/build/fingerprint_test.go +++ b/internal/build/fingerprint_test.go @@ -135,10 +135,9 @@ func TestManifestBuilder_EmptySections(t *testing.T) { m := newManifestBuilder() content := m.Build() - // Empty sections should not be written - expected := `` - if content != expected { - t.Errorf("unexpected empty manifest:\ngot:\n%s\nwant:\n%s", content, expected) + // Empty sections should not be written. + if content != "" { + t.Errorf("unexpected empty manifest:\ngot:\n%s", content) } // Should still produce a valid fingerprint diff --git a/internal/build/module_hook_test.go b/internal/build/module_hook_test.go index 99d2bde828..d91def525d 100644 --- a/internal/build/module_hook_test.go +++ b/internal/build/module_hook_test.go @@ -9,6 +9,7 @@ import ( func TestModuleHookReceivesMainPackageModule(t *testing.T) { conf := NewDefaultConf(ModeGen) + conf.Parallel = 2 counts := make(map[string]int) snapshots := make(map[string]string) diff --git a/internal/goflags/flagfile.go b/internal/goflags/flagfile.go index e8e4adf975..3586213e4e 100644 --- a/internal/goflags/flagfile.go +++ b/internal/goflags/flagfile.go @@ -27,6 +27,7 @@ var argumentListFlagNames = [...]string{ "gcflags", "gccgoflags", "ldflags", + "p", "toolexec", } diff --git a/internal/goflags/gobuild.go b/internal/goflags/gobuild.go index 6f38f42d9f..7d10074f1f 100644 --- a/internal/goflags/gobuild.go +++ b/internal/goflags/gobuild.go @@ -16,7 +16,13 @@ package goflags -import "github.com/goplus/llgo/internal/build" +import ( + "fmt" + "strconv" + "strings" + + "github.com/goplus/llgo/internal/build" +) // ApplyBuildFlags validates and appends normalized Go build flags, and maps // the supported compiler and linker semantics into typed build configuration. @@ -34,12 +40,37 @@ func ApplyBuildFlags(conf *build.Config, args []string) error { if err != nil { return err } + parallel, parallelSet, err := parseBuildParallel(all) + if err != nil { + return err + } next := *conf next.GoBuildFlags = all applyFrontendGCFlags(&next) + if parallelSet { + next.Parallel = parallel + } if linkFlags.Present { next.LinkOptions = linkFlags.Options } *conf = next return nil } + +// parseBuildParallel extracts Go's -p build concurrency flag after it has +// been normalized. Keeping it in GoBuildFlags still lets go/packages apply the +// same setting while Config.Parallel controls LLGo's own build stages. +func parseBuildParallel(flags []string) (parallel int, present bool, err error) { + for _, flag := range flags { + value, ok := strings.CutPrefix(flag, "-p=") + if !ok { + continue + } + parallel, err = strconv.Atoi(value) + if err != nil || parallel <= 0 { + return 0, false, fmt.Errorf("-p must be a positive integer, got %q", value) + } + present = true + } + return parallel, present, nil +} diff --git a/internal/goflags/gobuild_test.go b/internal/goflags/gobuild_test.go index f901e22f72..de809b7557 100644 --- a/internal/goflags/gobuild_test.go +++ b/internal/goflags/gobuild_test.go @@ -105,3 +105,28 @@ func TestApplyBuildFlagsFrontendGCFlagSemantics(t *testing.T) { }) } } + +func TestApplyBuildFlagsParallelism(t *testing.T) { + conf := &build.Config{} + if err := ApplyBuildFlags(conf, []string{"--p", "3"}); err != nil { + t.Fatal(err) + } + if conf.Parallel != 3 { + t.Fatalf("Parallel = %d, want 3", conf.Parallel) + } + if !reflect.DeepEqual(conf.GoBuildFlags, []string{"-p=3"}) { + t.Fatalf("GoBuildFlags = %#v, want [-p=3]", conf.GoBuildFlags) + } +} + +func TestApplyBuildFlagsRejectsInvalidParallelismAtomically(t *testing.T) { + conf := &build.Config{Parallel: 2, GoBuildFlags: []string{"-tags=existing"}} + want := *conf + want.GoBuildFlags = append([]string(nil), conf.GoBuildFlags...) + if err := ApplyBuildFlags(conf, []string{"-p=0"}); err == nil { + t.Fatal("ApplyBuildFlags succeeded, want error") + } + if !reflect.DeepEqual(*conf, want) { + t.Fatalf("configuration changed on error:\n got %+v\nwant %+v", *conf, want) + } +} From 47e6788073282f4230a76bf8844e3450b596e5d7 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 07:37:52 +0800 Subject: [PATCH 02/23] build: address parallelism review feedback --- internal/build/build.go | 16 ++++++++++------ internal/goflags/flagfile.go | 13 ++++++++++++- internal/goflags/flagfile_test.go | 11 +++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 870c570d3d..84932e7360 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -181,7 +181,7 @@ type Config struct { PCLNModeSet bool AllowNoBody bool // allow declarations without bodies, as go tool compile does // Parallel is the maximum number of LLGo's internally parallel build tasks. - // Zero uses GOMAXPROCS, matching the Go command's default -p behavior. + // Zero uses GOMAXPROCS; command-line -p, like go build -p, must be >= 1. Parallel int // PthreadStackSize sets a custom stack size, in bytes, for pthread-backed @@ -284,7 +284,7 @@ const ( var llssaInitOnce sync.Once func (c *Config) parallelism() int { - if c != nil && c.Parallel > 0 { + if c.Parallel > 0 { return c.Parallel } return runtime.GOMAXPROCS(0) @@ -2067,16 +2067,20 @@ func buildSSAPkgs(ctx *context, entries []ssaBuildEntry) { return } unique := make([]ssaBuildEntry, 0, len(entries)) - seen := make(map[*ssa.Package]bool, len(entries)) + entryIndex := make(map[*ssa.Package]int, len(entries)) for _, entry := range entries { - if entry.pkg == nil || seen[entry.pkg] { + if entry.pkg == nil { continue } - seen[entry.pkg] = true + if index, exists := entryIndex[entry.pkg]; exists { + unique[index].fixOrder = unique[index].fixOrder || entry.fixOrder + continue + } + entryIndex[entry.pkg] = len(unique) unique = append(unique, entry) } workers := min(ctx.buildConf.parallelism(), len(unique)) - jobs := make(chan ssaBuildEntry) + jobs := make(chan ssaBuildEntry, len(unique)) var wg sync.WaitGroup for range workers { wg.Add(1) diff --git a/internal/goflags/flagfile.go b/internal/goflags/flagfile.go index 3586213e4e..48ad661fcd 100644 --- a/internal/goflags/flagfile.go +++ b/internal/goflags/flagfile.go @@ -31,6 +31,17 @@ var argumentListFlagNames = [...]string{ "toolexec", } +// wholeLineValueFlagNames is the subset whose unquoted values can themselves +// contain arbitrary flag-like words. Scalar flags such as -p still belong to +// argumentListFlagNames for normalization, but must not consume a whole line. +var wholeLineValueFlagNames = [...]string{ + "asmflags", + "gcflags", + "gccgoflags", + "ldflags", + "toolexec", +} + // ParseFlagFile parses a flags.txt-style file. Blank lines and full-line // comments are ignored. A line beginning with a Go flag whose value is itself // an argument list is kept intact, so unquoted values such as @@ -63,7 +74,7 @@ func ParseFlagFile(data string) ([]string, error) { } func wholeLineValueFlag(line string) (flag string, ok bool) { - for _, name := range argumentListFlagNames { + for _, name := range wholeLineValueFlagNames { for _, prefix := range []string{"-" + name + "=", "--" + name + "="} { value, found := strings.CutPrefix(line, prefix) if !found { diff --git a/internal/goflags/flagfile_test.go b/internal/goflags/flagfile_test.go index d7938f3e93..efda442073 100644 --- a/internal/goflags/flagfile_test.go +++ b/internal/goflags/flagfile_test.go @@ -107,3 +107,14 @@ func TestParseFlagFileErrors(t *testing.T) { } } } + +func TestParseFlagFileKeepsScalarParallelFlagSeparate(t *testing.T) { + got, err := ParseFlagFile("-p=4 -trimpath -tags=fast\n") + if err != nil { + t.Fatal(err) + } + want := []string{"-p=4", "-trimpath", "-tags=fast"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ParseFlagFile() = %#v, want %#v", got, want) + } +} From a8fa66cf39791c2445c6e35620bc5c4f8d2a3af5 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 01:30:51 +0800 Subject: [PATCH 03/23] build: model package build units explicitly --- internal/build/build.go | 60 ++++++++++------------ internal/build/package_build.go | 77 ++++++++++++++++++++++++++++ internal/build/package_build_test.go | 57 ++++++++++++++++++++ 3 files changed, 161 insertions(+), 33 deletions(-) create mode 100644 internal/build/package_build.go create mode 100644 internal/build/package_build_test.go diff --git a/internal/build/build.go b/internal/build/build.go index 84932e7360..789e002b7b 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -863,21 +863,22 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []*aPackage - var normalPkgs []*aPackage + var runtimePkgs []packageBuildSpec + var normalPkgs []packageBuildSpec for _, p := range pkgs { - if isRuntimePkg(p.PkgPath) { - runtimePkgs = append(runtimePkgs, p) + spec := newPackageBuildSpec(p) + if spec.runtime { + runtimePkgs = append(runtimePkgs, spec) } else { - normalPkgs = append(normalPkgs, p) + normalPkgs = append(normalPkgs, spec) } } var needRuntime, needPyInit bool // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, p := range normalPkgs { - result, err := buildOnePackage(ctx, p, verbose) + for _, spec := range normalPkgs { + result, err := buildOnePackage(ctx, spec, verbose) if err != nil { return nil, err } @@ -887,8 +888,8 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, p := range runtimePkgs { - if _, err := buildOnePackage(ctx, p, verbose); err != nil { + for _, spec := range runtimePkgs { + if _, err := buildOnePackage(ctx, spec, verbose); err != nil { return nil, err } } @@ -897,35 +898,30 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return pkgs, nil } -type packageBuildResult struct { - needRuntime bool - needPyInit bool -} - // buildOnePackage is the serial package pipeline. The stages are separated so // a future scheduler can parallelize only stages with isolated package state. -func buildOnePackage(ctx *context, aPkg *aPackage, verbose bool) (packageBuildResult, error) { +func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + aPkg := spec.pkg pkg := aPkg.Package if _, ok := ctx.built[pkg.ID]; ok { - return packageBuildResult{}, nil + return packageBuildResultFor(spec), nil } ctx.built[pkg.ID] = none{} - kind, param := cl.PkgKindOf(pkg.Types) - if kind == cl.PkgDeclOnly { + if spec.isDeclOnly() { pkg.ExportFile = "" - return packageBuildResult{}, nil + return packageBuildResultFor(spec), nil } - if (kind == cl.PkgLinkIR || kind == cl.PkgLinkExtern || kind == cl.PkgPyModule) && len(pkg.GoFiles) == 0 { + if spec.isLinkOnly() && !spec.hasSource() { pkg.ExportFile = "" - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) + if spec.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } - return packageBuildResult{}, nil + return packageBuildResultFor(spec), nil } if err := ctx.collectFingerprint(aPkg); err != nil { - return packageBuildResult{}, err + return packageBuildResultFor(spec), err } ctx.tryLoadFromCache(aPkg) if verbose { @@ -936,28 +932,26 @@ func buildOnePackage(ctx *context, aPkg *aPackage, verbose bool) (packageBuildRe fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) } if err := buildPkg(ctx, aPkg, verbose); err != nil { - return packageBuildResult{}, err + return packageBuildResultFor(spec), err } - result := packageBuildResult{} - if kind != cl.PkgLinkIR && kind != cl.PkgLinkExtern && kind != cl.PkgPyModule { + if spec.needsRuntimeSignals() { aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) - result.needRuntime = aPkg.NeedRt - result.needPyInit = aPkg.NeedPyInit } + result := packageBuildResultFor(spec) if aPkg.CacheHit { return result, nil } if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return packageBuildResult{}, err + return result, err } - if kind == cl.PkgLinkExtern { - appendExternalLinkArgs(ctx, aPkg, param) + if spec.kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) } - return result, nil + return packageBuildResultFor(spec), nil } func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { diff --git a/internal/build/package_build.go b/internal/build/package_build.go new file mode 100644 index 0000000000..727267a813 --- /dev/null +++ b/internal/build/package_build.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import "github.com/goplus/llgo/cl" + +// packageBuildSpec is the immutable scheduler input for one package. It keeps +// package classification out of the execution loop, so a later DAG scheduler +// can choose work without reinterpreting frontend metadata. +type packageBuildSpec struct { + pkg *aPackage + kind int + kindParam string + runtime bool +} + +func newPackageBuildSpec(pkg *aPackage) packageBuildSpec { + kind, kindParam := cl.PkgKindOf(pkg.Types) + return packageBuildSpec{ + pkg: pkg, + kind: kind, + kindParam: kindParam, + runtime: isRuntimePkg(pkg.PkgPath), + } +} + +func (s packageBuildSpec) isDeclOnly() bool { + return s.kind == cl.PkgDeclOnly +} + +func (s packageBuildSpec) isLinkOnly() bool { + return s.kind == cl.PkgLinkIR || s.kind == cl.PkgLinkExtern || s.kind == cl.PkgPyModule +} + +func (s packageBuildSpec) hasSource() bool { + return len(s.pkg.GoFiles) > 0 +} + +func (s packageBuildSpec) needsRuntimeSignals() bool { + return !s.isLinkOnly() && !s.isDeclOnly() +} + +// packageBuildResult carries the observable output of a serial package build. +// Subsequent scheduler PRs can pass this value between worker and finalization +// stages without exposing the mutable aPackage implementation details. +type packageBuildResult struct { + spec packageBuildSpec + cacheHit bool + archiveFile string + needRuntime bool + needPyInit bool +} + +func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { + pkg := spec.pkg + return packageBuildResult{ + spec: spec, + cacheHit: pkg.CacheHit, + archiveFile: pkg.ArchiveFile, + needRuntime: pkg.NeedRt, + needPyInit: pkg.NeedPyInit, + } +} diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go new file mode 100644 index 0000000000..714525c4de --- /dev/null +++ b/internal/build/package_build_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/types" + "testing" + + "github.com/goplus/llgo/internal/packages" +) + +func TestPackageBuildSpecAndResult(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + PkgPath: "example.com/p", + GoFiles: []string{"p.go"}, + Types: types.NewPackage("example.com/p", "p"), + }, NeedRt: true, NeedPyInit: true, CacheHit: true, ArchiveFile: "p.a"} + spec := newPackageBuildSpec(pkg) + if spec.isDeclOnly() || spec.isLinkOnly() || !spec.hasSource() || spec.runtime || !spec.needsRuntimeSignals() { + t.Fatalf("unexpected normal package spec: %+v", spec) + } + result := packageBuildResultFor(spec) + if !result.cacheHit || result.archiveFile != "p.a" || !result.needRuntime || !result.needPyInit { + t.Fatalf("unexpected package result: %+v", result) + } +} + +func TestPackageBuildSpecSpecialKinds(t *testing.T) { + decl := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + PkgPath: "unsafe", + Types: types.Unsafe, + }}) + if !decl.isDeclOnly() || decl.needsRuntimeSignals() { + t.Fatalf("unexpected declaration-only spec: %+v", decl) + } + runtime := newPackageBuildSpec(&aPackage{Package: &packages.Package{ + PkgPath: "runtime", + Types: types.NewPackage("runtime", "runtime"), + }}) + if !runtime.runtime { + t.Fatalf("runtime package was not marked runtime: %+v", runtime) + } +} From c28caf004bce46c36eebfd42be89709c8516496e Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 07:39:33 +0800 Subject: [PATCH 04/23] test: use the llgo runtime path in build spec coverage --- internal/build/package_build_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 714525c4de..f2360ed6bc 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -20,6 +20,7 @@ import ( "go/types" "testing" + "github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/packages" ) @@ -48,8 +49,8 @@ func TestPackageBuildSpecSpecialKinds(t *testing.T) { t.Fatalf("unexpected declaration-only spec: %+v", decl) } runtime := newPackageBuildSpec(&aPackage{Package: &packages.Package{ - PkgPath: "runtime", - Types: types.NewPackage("runtime", "runtime"), + PkgPath: env.LLGoRuntimePkg, + Types: types.NewPackage(env.LLGoRuntimePkg, "runtime"), }}) if !runtime.runtime { t.Fatalf("runtime package was not marked runtime: %+v", runtime) From 475f24b691e3896abf870420e077647621042b38 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 01:33:51 +0800 Subject: [PATCH 05/23] build: split package execution stages --- internal/build/build.go | 49 +++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 789e002b7b..f81d82888d 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -901,27 +901,39 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er // buildOnePackage is the serial package pipeline. The stages are separated so // a future scheduler can parallelize only stages with isolated package state. func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + skip, err := preflightPackageBuild(ctx, spec, verbose) + if err != nil || skip { + return packageBuildResultFor(spec), err + } + if err := executePackageBuild(ctx, spec, verbose); err != nil { + return packageBuildResultFor(spec), err + } + return finalizePackageBuild(ctx, spec, verbose) +} + +// preflightPackageBuild performs package classification, cache lookup, and +// other work that does not create or transform an LLVM module. It returns +// skip for packages with no executable build stage. +func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (skip bool, err error) { aPkg := spec.pkg pkg := aPkg.Package if _, ok := ctx.built[pkg.ID]; ok { - return packageBuildResultFor(spec), nil + return true, nil } ctx.built[pkg.ID] = none{} - if spec.isDeclOnly() { pkg.ExportFile = "" - return packageBuildResultFor(spec), nil + return true, nil } if spec.isLinkOnly() && !spec.hasSource() { pkg.ExportFile = "" if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } - return packageBuildResultFor(spec), nil + return true, nil } - if err := ctx.collectFingerprint(aPkg); err != nil { - return packageBuildResultFor(spec), err + return false, err } ctx.tryLoadFromCache(aPkg) if verbose { @@ -931,25 +943,38 @@ func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (package } fmt.Fprintf(os.Stderr, "CACHE %s: %s\n", status, pkg.PkgPath) } + return false, nil +} + +// executePackageBuild creates the frontend module and, on a cache miss, runs +// the serial LLVM backend. The backend remains deliberately serial until it +// owns an isolated LLVM context. +func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) error { + aPkg := spec.pkg if err := buildPkg(ctx, aPkg, verbose); err != nil { - return packageBuildResultFor(spec), err + return err } - if spec.needsRuntimeSignals() { aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) } - result := packageBuildResultFor(spec) + return nil +} + +// finalizePackageBuild publishes an archive and cache metadata after backend +// execution. Cache hits already carry their archive and metadata. +func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { + aPkg := spec.pkg if aPkg.CacheHit { - return result, nil + return packageBuildResultFor(spec), nil } if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return result, err + return packageBuildResultFor(spec), err } if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } if err := ctx.saveToCache(aPkg); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } return packageBuildResultFor(spec), nil } From 1873e94c8dbdc7aeb0dad6a31f2c15fb90ebc949 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 01:36:27 +0800 Subject: [PATCH 06/23] build: plan package dependency levels --- internal/build/build.go | 7 +- internal/build/package_build.go | 100 ++++++++++++++++++++++++++- internal/build/package_build_test.go | 60 ++++++++++++++++ 3 files changed, 164 insertions(+), 3 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index f81d82888d..9ada46e30f 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -862,11 +862,14 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { + plan, err := newPackageBuildPlan(pkgs) + if err != nil { + return nil, err + } // Split packages into runtime tree vs others so we can defer runtime build. var runtimePkgs []packageBuildSpec var normalPkgs []packageBuildSpec - for _, p := range pkgs { - spec := newPackageBuildSpec(p) + for _, spec := range plan.specs { if spec.runtime { runtimePkgs = append(runtimePkgs, spec) } else { diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 727267a813..6de5e9f399 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -16,7 +16,12 @@ package build -import "github.com/goplus/llgo/cl" +import ( + "fmt" + "sort" + + "github.com/goplus/llgo/cl" +) // packageBuildSpec is the immutable scheduler input for one package. It keeps // package classification out of the execution loop, so a later DAG scheduler @@ -75,3 +80,96 @@ func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { needPyInit: pkg.NeedPyInit, } } + +// packageBuildPlan is the immutable dependency graph consumed by a future +// scheduler. specs retains the existing deterministic execution order until +// the LLVM backend is made worker-local; levels records the safe ready sets. +type packageBuildPlan struct { + specs []packageBuildSpec + byID map[string]packageBuildSpec + deps map[string][]string + levels [][]packageBuildSpec +} + +func newPackageBuildPlan(pkgs []*aPackage) (*packageBuildPlan, error) { + plan := &packageBuildPlan{ + specs: make([]packageBuildSpec, 0, len(pkgs)), + byID: make(map[string]packageBuildSpec, len(pkgs)), + deps: make(map[string][]string, len(pkgs)), + } + for _, pkg := range pkgs { + spec := newPackageBuildSpec(pkg) + id := spec.pkg.ID + if _, exists := plan.byID[id]; exists { + return nil, fmt.Errorf("duplicate package build spec for %s", id) + } + plan.specs = append(plan.specs, spec) + plan.byID[id] = spec + } + for _, spec := range plan.specs { + id := spec.pkg.ID + for _, dep := range effectiveDependencies(spec.pkg) { + if _, inPlan := plan.byID[dep.ID]; inPlan { + plan.deps[id] = append(plan.deps[id], dep.ID) + } + } + sort.Strings(plan.deps[id]) + } + levels, err := plan.readyLevels() + if err != nil { + return nil, err + } + plan.levels = levels + return plan, nil +} + +func (p *packageBuildPlan) readyLevels() ([][]packageBuildSpec, error) { + remaining := make(map[string]int, len(p.specs)) + dependents := make(map[string][]string, len(p.specs)) + for _, spec := range p.specs { + id := spec.pkg.ID + remaining[id] = len(p.deps[id]) + for _, dep := range p.deps[id] { + dependents[dep] = append(dependents[dep], id) + } + } + for _, ids := range dependents { + sort.Strings(ids) + } + + ready := make([]string, 0, len(p.specs)) + for id, count := range remaining { + if count == 0 { + ready = append(ready, id) + } + } + sort.Strings(ready) + levels := make([][]packageBuildSpec, 0, len(p.specs)) + for len(ready) > 0 { + ids := ready + ready = nil + level := make([]packageBuildSpec, 0, len(ids)) + for _, id := range ids { + level = append(level, p.byID[id]) + for _, dependent := range dependents[id] { + remaining[dependent]-- + if remaining[dependent] == 0 { + ready = append(ready, dependent) + } + } + } + sort.Strings(ready) + levels = append(levels, level) + } + if len(levels) == 0 && len(p.specs) == 0 { + return levels, nil + } + count := 0 + for _, level := range levels { + count += len(level) + } + if count != len(p.specs) { + return nil, fmt.Errorf("package build dependency cycle") + } + return levels, nil +} diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index f2360ed6bc..d713034883 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -18,6 +18,7 @@ package build import ( "go/types" + "reflect" "testing" "github.com/goplus/llgo/internal/env" @@ -40,6 +41,65 @@ func TestPackageBuildSpecAndResult(t *testing.T) { } } +func TestPackageBuildPlanReadyLevels(t *testing.T) { + leaf := planPackage("leaf") + left := planPackage("left", leaf.Package) + right := planPackage("right", leaf.Package) + root := planPackage("root", left.Package, right.Package) + plan, err := newPackageBuildPlan([]*aPackage{root, right, left, leaf}) + if err != nil { + t.Fatal(err) + } + var levels [][]string + for _, level := range plan.levels { + ids := make([]string, len(level)) + for i, spec := range level { + ids[i] = spec.pkg.ID + } + levels = append(levels, ids) + } + if want := [][]string{{"leaf"}, {"left", "right"}, {"root"}}; !reflect.DeepEqual(levels, want) { + t.Fatalf("ready levels = %v, want %v", levels, want) + } +} + +func TestPackageBuildPlanIncludesAlternateDependencies(t *testing.T) { + baseDep := planPackage("base") + altDep := planPackage("alt") + pkg := planPackage("pkg", baseDep.Package) + pkg.AltPkg = &packages.Cached{Package: &packages.Package{ID: "patch/pkg", Imports: map[string]*packages.Package{"alt": altDep.Package}}} + plan, err := newPackageBuildPlan([]*aPackage{pkg, baseDep, altDep}) + if err != nil { + t.Fatal(err) + } + if got, want := plan.deps["pkg"], []string{"alt", "base"}; !reflect.DeepEqual(got, want) { + t.Fatalf("plan dependencies = %v, want %v", got, want) + } +} + +func TestPackageBuildPlanRejectsCycles(t *testing.T) { + a := planPackage("a") + b := planPackage("b") + a.Imports = map[string]*packages.Package{"b": b.Package} + b.Imports = map[string]*packages.Package{"a": a.Package} + if _, err := newPackageBuildPlan([]*aPackage{a, b}); err == nil { + t.Fatal("newPackageBuildPlan succeeded, want cycle error") + } +} + +func planPackage(id string, imports ...*packages.Package) *aPackage { + depMap := make(map[string]*packages.Package, len(imports)) + for _, dep := range imports { + depMap[dep.ID] = dep + } + return &aPackage{Package: &packages.Package{ + ID: id, + PkgPath: "example.com/" + id, + Imports: depMap, + Types: types.NewPackage("example.com/"+id, id), + }} +} + func TestPackageBuildSpecSpecialKinds(t *testing.T) { decl := newPackageBuildSpec(&aPackage{Package: &packages.Package{ PkgPath: "unsafe", From da465b3185ccf08f94adc915c9af4fd9e87a4630 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 07:40:16 +0800 Subject: [PATCH 07/23] build: identify package dependency cycles --- internal/build/package_build.go | 10 +++++++++- internal/build/package_build_test.go | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 6de5e9f399..7da55c003f 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -19,6 +19,7 @@ package build import ( "fmt" "sort" + "strings" "github.com/goplus/llgo/cl" ) @@ -169,7 +170,14 @@ func (p *packageBuildPlan) readyLevels() ([][]packageBuildSpec, error) { count += len(level) } if count != len(p.specs) { - return nil, fmt.Errorf("package build dependency cycle") + cycle := make([]string, 0, len(p.specs)-count) + for id, unresolved := range remaining { + if unresolved > 0 { + cycle = append(cycle, id) + } + } + sort.Strings(cycle) + return nil, fmt.Errorf("package build dependency cycle involving %s", strings.Join(cycle, ", ")) } return levels, nil } diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index d713034883..59e7ed275a 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -19,6 +19,7 @@ package build import ( "go/types" "reflect" + "strings" "testing" "github.com/goplus/llgo/internal/env" @@ -84,6 +85,8 @@ func TestPackageBuildPlanRejectsCycles(t *testing.T) { b.Imports = map[string]*packages.Package{"a": a.Package} if _, err := newPackageBuildPlan([]*aPackage{a, b}); err == nil { t.Fatal("newPackageBuildPlan succeeded, want cycle error") + } else if !strings.Contains(err.Error(), "a, b") { + t.Fatalf("cycle error = %q, want package IDs", err) } } From d0e6e553361ddfc9734fbd9a5d188413c7feeca6 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 01:40:36 +0800 Subject: [PATCH 08/23] build: parallelize cache preflight by dependency level --- internal/build/build.go | 50 +++++++++++++++++++-------------- internal/build/collect.go | 36 +++++++++++++----------- internal/build/package_build.go | 48 +++++++++++++++++++++++++++++++ internal/build/plan9asm.go | 2 ++ 4 files changed, 99 insertions(+), 37 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 9ada46e30f..833933fa08 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -519,15 +519,14 @@ func Do(args []string, conf *Config) ([]Package, error) { ctx := &context{env: env, conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, patches: patches, callerTracking: cl.NewCallerTracking(), built: make(map[string]none), initial: initial, mode: mode, - fingerprinting: make(map[string]bool), - pkgs: map[*packages.Package]Package{}, - pkgByID: map[string]Package{}, - output: output, - passOpt: passOpt, - buildConf: conf, - crossCompile: export, - cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), - sfilesCache: make(map[string][]string), + pkgs: map[*packages.Package]Package{}, + pkgByID: map[string]Package{}, + output: output, + passOpt: passOpt, + buildConf: conf, + crossCompile: export, + cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + sfilesCache: make(map[string][]string), } defer ctx.closePackageMetas() ctx.initializePackageBuildState() @@ -756,7 +755,7 @@ type context struct { patches cl.Patches callerTracking *cl.CallerTracking built map[string]none - fingerprinting map[string]bool + builtMu sync.Mutex initial []*packages.Package pkgs map[*packages.Package]Package // cache for lookup pkgByID map[string]Package // cache for lookup by pkg.ID @@ -774,11 +773,14 @@ type context struct { // Cache related fields cacheManager *cacheManager + cacheManagerMu sync.Mutex llvmVersion string llvmVersionReady bool + llvmVersionMu sync.Mutex // go list derived file lists (SFiles, etc.) sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths + sfilesMu sync.Mutex // plan9asm package policy parsed from env. plan9asmOnce sync.Once @@ -866,6 +868,10 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er if err != nil { return nil, err } + preflights, err := preflightPackageBuilds(ctx, plan, verbose) + if err != nil { + return nil, err + } // Split packages into runtime tree vs others so we can defer runtime build. var runtimePkgs []packageBuildSpec var normalPkgs []packageBuildSpec @@ -881,7 +887,7 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er // Build non-runtime packages first, so we know whether runtime is actually needed. for _, spec := range normalPkgs { - result, err := buildOnePackage(ctx, spec, verbose) + result, err := buildPreflightedPackage(ctx, preflights[spec.pkg.ID], verbose) if err != nil { return nil, err } @@ -892,7 +898,7 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { for _, spec := range runtimePkgs { - if _, err := buildOnePackage(ctx, spec, verbose); err != nil { + if _, err := buildPreflightedPackage(ctx, preflights[spec.pkg.ID], verbose); err != nil { return nil, err } } @@ -901,17 +907,16 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er return pkgs, nil } -// buildOnePackage is the serial package pipeline. The stages are separated so -// a future scheduler can parallelize only stages with isolated package state. -func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (packageBuildResult, error) { - skip, err := preflightPackageBuild(ctx, spec, verbose) - if err != nil || skip { - return packageBuildResultFor(spec), err +// buildPreflightedPackage runs the serial module/backend/finalization stages +// after parallel preflight has completed for the package's dependency level. +func buildPreflightedPackage(ctx *context, preflight packagePreflight, verbose bool) (packageBuildResult, error) { + if preflight.skip { + return packageBuildResultFor(preflight.spec), nil } - if err := executePackageBuild(ctx, spec, verbose); err != nil { - return packageBuildResultFor(spec), err + if err := executePackageBuild(ctx, preflight.spec, verbose); err != nil { + return packageBuildResultFor(preflight.spec), err } - return finalizePackageBuild(ctx, spec, verbose) + return finalizePackageBuild(ctx, preflight.spec, verbose) } // preflightPackageBuild performs package classification, cache lookup, and @@ -920,10 +925,13 @@ func buildOnePackage(ctx *context, spec packageBuildSpec, verbose bool) (package func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (skip bool, err error) { aPkg := spec.pkg pkg := aPkg.Package + ctx.builtMu.Lock() if _, ok := ctx.built[pkg.ID]; ok { + ctx.builtMu.Unlock() return true, nil } ctx.built[pkg.ID] = none{} + ctx.builtMu.Unlock() if spec.isDeclOnly() { pkg.ExportFile = "" return true, nil diff --git a/internal/build/collect.go b/internal/build/collect.go index 7a41e427ae..ff099f60f0 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -34,17 +34,18 @@ import ( // collectFingerprint collects all inputs and generates fingerprint for a package. func (c *context) collectFingerprint(pkg *aPackage) error { + return c.collectFingerprintWithStack(pkg, make(map[string]bool)) +} + +func (c *context) collectFingerprintWithStack(pkg *aPackage, fingerprinting map[string]bool) error { if pkg.Manifest != "" && pkg.Fingerprint != "" { return nil } - if c.fingerprinting == nil { - c.fingerprinting = make(map[string]bool) - } - if c.fingerprinting[pkg.ID] { + if fingerprinting[pkg.ID] { return fmt.Errorf("fingerprint cycle detected for %s", pkg.ID) } - c.fingerprinting[pkg.ID] = true - defer delete(c.fingerprinting, pkg.ID) + fingerprinting[pkg.ID] = true + defer delete(fingerprinting, pkg.ID) m := newManifestBuilder() @@ -60,7 +61,7 @@ func (c *context) collectFingerprint(pkg *aPackage) error { } // Dependency section - if err := c.collectDependencyInputs(m, pkg); err != nil { + if err := c.collectDependencyInputs(m, pkg, fingerprinting); err != nil { return err } @@ -193,9 +194,9 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error } // collectDependencyInputs adds dependency fingerprints/versions into manifest. -func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) error { +func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage, fingerprinting map[string]bool) error { for _, dep := range effectiveDependencies(pkg) { - depEntry, err := c.dependencyFingerprint(dep) + depEntry, err := c.dependencyFingerprint(dep, fingerprinting) if err != nil { return err } @@ -212,15 +213,14 @@ func (c *context) initializePackageBuildState() { if c.sfilesCache == nil { c.sfilesCache = make(map[string][]string) } - if !cacheEnabled() { - return - } - c.cacheManager = newCacheManager() c.llvmVersion = detectLLVMVersion(c) c.llvmVersionReady = true + if cacheEnabled() { + c.cacheManager = newCacheManager() + } } -func (c *context) dependencyFingerprint(dep *packages.Package) (depEntry, error) { +func (c *context) dependencyFingerprint(dep *packages.Package, fingerprinting map[string]bool) (depEntry, error) { entry := depEntry{ID: dep.ID} if v := moduleVersion(dep.Module); v != "" { entry.Version = v @@ -230,7 +230,7 @@ func (c *context) dependencyFingerprint(dep *packages.Package) (depEntry, error) if c.pkgByID != nil { if aDep, ok := c.pkgByID[dep.ID]; ok { if aDep.Fingerprint == "" { - if err := c.collectFingerprint(aDep); err != nil { + if err := c.collectFingerprintWithStack(aDep, fingerprinting); err != nil { return entry, fmt.Errorf("collect fingerprint for %s: %w", dep.ID, err) } } @@ -240,7 +240,7 @@ func (c *context) dependencyFingerprint(dep *packages.Package) (depEntry, error) } temp := &aPackage{Package: dep} - if err := c.collectFingerprint(temp); err != nil { + if err := c.collectFingerprintWithStack(temp, fingerprinting); err != nil { return entry, fmt.Errorf("collect fingerprint for %s: %w", dep.ID, err) } entry.Fingerprint = temp.Fingerprint @@ -263,6 +263,8 @@ func moduleVersion(mod *gopackages.Module) string { // getLLVMVersion returns the cached LLVM version or detects it. func (c *context) getLLVMVersion() string { + c.llvmVersionMu.Lock() + defer c.llvmVersionMu.Unlock() if c.llvmVersionReady { return c.llvmVersion } @@ -314,6 +316,8 @@ func targetTriple(goos, goarch, llvmTarget, targetABI string) string { // ensureCacheManager creates cacheManager if not exists. func (c *context) ensureCacheManager() *cacheManager { + c.cacheManagerMu.Lock() + defer c.cacheManagerMu.Unlock() if c.cacheManager == nil { c.cacheManager = newCacheManager() } diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 7da55c003f..ca23153608 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -20,6 +20,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/goplus/llgo/cl" ) @@ -34,6 +35,53 @@ type packageBuildSpec struct { runtime bool } +type packagePreflight struct { + spec packageBuildSpec + skip bool +} + +// preflightPackageBuilds runs only cache/input work for one dependency level at +// a time. Every dependency's fingerprint is therefore complete before a worker +// reads it, while unrelated packages share the configured -p bound. +func preflightPackageBuilds(ctx *context, plan *packageBuildPlan, verbose bool) (map[string]packagePreflight, error) { + preflights := make(map[string]packagePreflight, len(plan.specs)) + for _, level := range plan.levels { + workers := min(ctx.buildConf.parallelism(), len(level)) + jobs := make(chan int) + type result struct { + index int + skip bool + err error + } + results := make(chan result, len(level)) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + skip, err := preflightPackageBuild(ctx, level[index], verbose) + results <- result{index: index, skip: skip, err: err} + } + }() + } + for index := range level { + jobs <- index + } + close(jobs) + wg.Wait() + close(results) + for result := range results { + if result.err != nil { + return nil, result.err + } + spec := level[result.index] + preflights[spec.pkg.ID] = packagePreflight{spec: spec, skip: result.skip} + } + } + return preflights, nil +} + func newPackageBuildSpec(pkg *aPackage) packageBuildSpec { kind, kindParam := cl.PkgKindOf(pkg.Types) return packageBuildSpec{ diff --git a/internal/build/plan9asm.go b/internal/build/plan9asm.go index 57938f22c5..9794feebd2 100644 --- a/internal/build/plan9asm.go +++ b/internal/build/plan9asm.go @@ -394,6 +394,8 @@ func pkgSFiles(ctx *context, pkg *packages.Package) ([]string, error) { } } + ctx.sfilesMu.Lock() + defer ctx.sfilesMu.Unlock() if ctx.sfilesCache == nil { ctx.sfilesCache = make(map[string][]string) } From 1666b2d9c4d1650c2a6077b6763aa9899138f4b1 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 07:42:46 +0800 Subject: [PATCH 09/23] build: fix parallel preflight synchronization --- internal/build/build.go | 4 +-- internal/build/collect.go | 8 +++--- internal/build/package_build.go | 6 ++--- internal/build/package_build_test.go | 21 +++++++++++++++ internal/build/plan9asm.go | 39 +++++++++++++++++++--------- 5 files changed, 56 insertions(+), 22 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 833933fa08..cbd4e5317d 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -934,14 +934,14 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s ctx.builtMu.Unlock() if spec.isDeclOnly() { pkg.ExportFile = "" - return true, nil + return true, ctx.collectFingerprint(aPkg) } if spec.isLinkOnly() && !spec.hasSource() { pkg.ExportFile = "" if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } - return true, nil + return true, ctx.collectFingerprint(aPkg) } if err := ctx.collectFingerprint(aPkg); err != nil { return false, err diff --git a/internal/build/collect.go b/internal/build/collect.go index ff099f60f0..a27f83879f 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -206,15 +206,13 @@ func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage, fin return nil } -// initializePackageBuildState initializes the mutable state used by the -// package pipeline before any scheduler is introduced. This makes ownership -// explicit and avoids lazy first-use writes becoming data races later. +// initializePackageBuildState initializes mutable package pipeline state that +// has to exist before parallel preflight begins. LLVM version discovery stays +// lazy and is synchronized by getLLVMVersion. func (c *context) initializePackageBuildState() { if c.sfilesCache == nil { c.sfilesCache = make(map[string][]string) } - c.llvmVersion = detectLLVMVersion(c) - c.llvmVersionReady = true if cacheEnabled() { c.cacheManager = newCacheManager() } diff --git a/internal/build/package_build.go b/internal/build/package_build.go index ca23153608..4ee5f925b8 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -130,9 +130,9 @@ func packageBuildResultFor(spec packageBuildSpec) packageBuildResult { } } -// packageBuildPlan is the immutable dependency graph consumed by a future -// scheduler. specs retains the existing deterministic execution order until -// the LLVM backend is made worker-local; levels records the safe ready sets. +// packageBuildPlan is the immutable dependency graph. levels drives bounded +// parallel preflight today; specs retains the deterministic serial order for +// the LLVM backend until it owns worker-local program state. type packageBuildPlan struct { specs []packageBuildSpec byID map[string]packageBuildSpec diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 59e7ed275a..7b3e26ace1 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -119,3 +119,24 @@ func TestPackageBuildSpecSpecialKinds(t *testing.T) { t.Fatalf("runtime package was not marked runtime: %+v", runtime) } } + +func TestPreflightFingerprintsSkippedPackage(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ + ID: "unsafe", + PkgPath: "unsafe", + Types: types.Unsafe, + }} + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64", ForceRebuild: true}, + built: make(map[string]none), + llvmVersionReady: true, + } + skip, err := preflightPackageBuild(ctx, newPackageBuildSpec(pkg), false) + if err != nil { + t.Fatal(err) + } + if !skip || pkg.Fingerprint == "" || pkg.Manifest == "" { + t.Fatalf("skipped package was not fingerprinted: skip=%v fingerprint=%q manifest=%q", skip, pkg.Fingerprint, pkg.Manifest) + } +} diff --git a/internal/build/plan9asm.go b/internal/build/plan9asm.go index 9794feebd2..88b2cf1595 100644 --- a/internal/build/plan9asm.go +++ b/internal/build/plan9asm.go @@ -394,12 +394,7 @@ func pkgSFiles(ctx *context, pkg *packages.Package) ([]string, error) { } } - ctx.sfilesMu.Lock() - defer ctx.sfilesMu.Unlock() - if ctx.sfilesCache == nil { - ctx.sfilesCache = make(map[string][]string) - } - if v, ok := ctx.sfilesCache[pkg.ID]; ok { + if v, ok := getCachedSFiles(ctx, pkg.ID); ok { return v, nil } @@ -447,21 +442,41 @@ func pkgSFiles(ctx *context, pkg *packages.Package) ([]string, error) { stub := filepath.Join(lp.Dir, "chacha8_stub.s") if _, err := os.Stat(stub); err == nil { paths := []string{stub} - ctx.sfilesCache[pkg.ID] = paths - return paths, nil + return cacheSFiles(ctx, pkg.ID, paths), nil } } // Embedded ARM targets currently reuse GOOS=linux metadata, but they do not // have a Linux syscall surface. Skip syscall asm in that mode so embedded // builds do not inherit Linux/ARM-specific frame layouts. if shouldSkipPlan9AsmSFilesForTarget(ctx.buildConf, pkg.PkgPath) { - ctx.sfilesCache[pkg.ID] = nil - return nil, nil + return cacheSFiles(ctx, pkg.ID, nil), nil } paths := selectedSFiles(lp.Dir, lp.SFiles) - ctx.sfilesCache[pkg.ID] = paths - return paths, nil + return cacheSFiles(ctx, pkg.ID, paths), nil +} + +func getCachedSFiles(ctx *context, pkgID string) ([]string, bool) { + ctx.sfilesMu.Lock() + defer ctx.sfilesMu.Unlock() + if ctx.sfilesCache == nil { + ctx.sfilesCache = make(map[string][]string) + } + paths, ok := ctx.sfilesCache[pkgID] + return paths, ok +} + +func cacheSFiles(ctx *context, pkgID string, paths []string) []string { + ctx.sfilesMu.Lock() + defer ctx.sfilesMu.Unlock() + if ctx.sfilesCache == nil { + ctx.sfilesCache = make(map[string][]string) + } + if cached, ok := ctx.sfilesCache[pkgID]; ok { + return cached + } + ctx.sfilesCache[pkgID] = paths + return paths } func selectedSFiles(dir string, files []string) []string { From d615a88cc6612666570f4d324131468eeefee291 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 08:01:15 +0800 Subject: [PATCH 10/23] build: snapshot package linker state --- internal/build/build.go | 81 +++++---- internal/build/build_test.go | 12 +- internal/build/collect.go | 23 ++- internal/build/collect_test.go | 76 ++++++++ internal/build/fingerprint.go | 7 +- internal/build/funcinfo_table.go | 37 ++-- internal/build/package_summary.go | 230 +++++++++++++++++++++++++ internal/build/package_summary_test.go | 86 +++++++++ 8 files changed, 487 insertions(+), 65 deletions(-) create mode 100644 internal/build/package_summary.go create mode 100644 internal/build/package_summary_test.go diff --git a/internal/build/build.go b/internal/build/build.go index cbd4e5317d..36d3196696 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -984,6 +984,9 @@ func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (pa if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } + // Snapshot every linker-facing LLVM fact before cache publication. The + // summary is the hand-off point to a future worker-local Program. + aPkg.Summary = summarizePackage(aPkg) if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } @@ -1235,35 +1238,40 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa linkedOrder = append(linkedOrder, aPkg) } }) + linkedSummaries := make([]*PackageSummary, len(linkedOrder)) + for i, aPkg := range linkedOrder { + if aPkg.Summary == nil { + return fmt.Errorf("package %s has no linker summary", aPkg.PkgPath) + } + linkedSummaries[i] = aPkg.Summary + } // packages.Visit with a post callback yields dependencies before importers. // Reverse that order so static archives are linked after the objects that use them. for i := len(linkedOrder) - 1; i >= 0; i-- { - aPkg := linkedOrder[i] - p := aPkg.Package + summary := linkedSummaries[i] // Defer linking runtime packages unless we actually need the runtime. - if isRuntimePkg(p.PkgPath) { - rtLinkArgs = append(rtLinkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - rtLinkInputs = append(rtLinkInputs, aPkg.ArchiveFile) + if isRuntimePkg(summary.PkgPath) { + rtLinkArgs = append(rtLinkArgs, summary.LinkArgs...) + if summary.ArchiveFile != "" { + rtLinkInputs = append(rtLinkInputs, summary.ArchiveFile) } continue } // Only let non-runtime packages influence whether runtime is needed. - need1, need2 := aPkg.isNeedRuntimeOrPyInit() - needRuntime = needRuntime || need1 - needPyInit = needPyInit || need2 - needAbiInit |= aPkg.LPkg.NeedAbiInit - for k, _ := range aPkg.LPkg.MethodByIndex { - methodByIndex[k] = none{} + needRuntime = needRuntime || summary.NeedRuntime + needPyInit = needPyInit || summary.NeedPyInit + needAbiInit |= summary.NeedAbiInit + for _, method := range summary.MethodByIndex { + methodByIndex[method] = none{} } - for k, _ := range aPkg.LPkg.MethodByName { - methodByName[k] = none{} + for _, method := range summary.MethodByName { + methodByName[method] = none{} } - linkArgs = append(linkArgs, aPkg.LinkArgs...) - if aPkg.ArchiveFile != "" { - archiveInputs = append(archiveInputs, aPkg.ArchiveFile) + linkArgs = append(linkArgs, summary.LinkArgs...) + if summary.ArchiveFile != "" { + archiveInputs = append(archiveInputs, summary.ArchiveFile) } } @@ -1280,9 +1288,9 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa var pcLineInfo []pcLineRecord var funcInfoStubs []funcInfoStubRecord if ctx.buildConf.PCLNMode != PCLNNone { - funcInfo = prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil) - pcLineInfo = collectPCLineInfo(linkedOrder) - funcInfoStubs = collectFuncInfoStubRecords(linkedOrder, funcInfo) + funcInfo = prepareFuncInfoTableRecords(collectFuncInfoSummaries(linkedSummaries), nil) + pcLineInfo = collectPCLineInfoSummaries(linkedSummaries) + funcInfoStubs = collectFuncInfoStubRecordsSummaries(linkedSummaries, funcInfo) } entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{ rtInit: needRuntime, @@ -1290,7 +1298,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa abiInit: needAbiInit, methodByIndex: methodByIndex, methodByName: methodByName, - abiSymbols: linkedModuleGlobals(linkedOrder), + abiSymbols: linkedPackageGlobals(linkedSummaries), funcInfo: funcInfo, pcLineInfo: pcLineInfo, funcInfoStubs: funcInfoStubs, @@ -1325,7 +1333,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } } } - linkArgs = append(linkArgs, cSharedExportArgs(ctx, linkedOrder)...) + linkArgs = append(linkArgs, cSharedExportArgs(ctx, linkedSummaries)...) err = linkObjFiles(ctx, outputPath, linkInputs, linkArgs, verbose) if err != nil { @@ -1336,19 +1344,22 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa } func linkedModuleGlobals(pkgs []Package) map[string]none { - if len(pkgs) == 0 { + return linkedPackageGlobals(summariesForPackages(pkgs)) +} + +func linkedPackageGlobals(summaries []*PackageSummary) map[string]none { + if len(summaries) == 0 { return nil } seen := make(map[string]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for g := pkg.LPkg.Module().FirstGlobal(); !g.IsNil(); g = gllvm.NextGlobal(g) { - if g.IsDeclaration() { - continue + for _, name := range summary.GlobalSymbols { + if name != "" { + seen[name] = none{} } - seen[g.Name()] = none{} } } return seen @@ -1421,16 +1432,16 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose // cSharedExportArgs keeps //export functions as shared-library link roots. The // functions live in package archives and otherwise remain unreferenced, so the // linker can omit both their object files and dynamic symbols. -func cSharedExportArgs(ctx *context, pkgs []*aPackage) []string { +func cSharedExportArgs(ctx *context, summaries []*PackageSummary) []string { if ctx == nil || ctx.buildConf == nil || ctx.buildConf.BuildMode != BuildModeCShared { return nil } exports := make(map[string]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, name := range pkg.LPkg.ExportFuncs() { + for _, name := range summary.CSharedExports { if name != "" { exports[name] = none{} } @@ -2015,6 +2026,10 @@ type aPackage struct { SSA *ssa.Package AltPkg *packages.Cached LPkg llssa.Package + // Summary is the immutable linker-facing result captured after the package + // module is compiled. Linkers must use it instead of reading LPkg so a + // future backend worker can release its LLVM program after object emission. + Summary *PackageSummary NeedRt bool NeedPyInit bool diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 85614585cd..105c4cec8a 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -645,22 +645,18 @@ func TestCSharedExportArgs(t *testing.T) { if got := cSharedExportArgs(nil, nil); got != nil { t.Fatalf("nil cSharedExportArgs = %v, want nil", got) } - prog := llssa.NewProgram(nil) - lpkg := prog.NewPackage("example.com/p", "example.com/p") - lpkg.SetExport("example.com/p.Z", "Zed") - lpkg.SetExport("example.com/p.A", "Add") - pkgs := []*aPackage{{LPkg: lpkg}} + summaries := []*PackageSummary{{CSharedExports: []string{"Add", "Zed"}}} ctx := &context{buildConf: &Config{BuildMode: BuildModeCShared, Goos: "linux"}} - if got, want := strings.Join(cSharedExportArgs(ctx, pkgs), " "), "-Wl,--undefined=Add -Wl,--undefined=Zed"; got != want { + if got, want := strings.Join(cSharedExportArgs(ctx, summaries), " "), "-Wl,--undefined=Add -Wl,--undefined=Zed"; got != want { t.Fatalf("linux cSharedExportArgs = %q, want %q", got, want) } ctx.buildConf.Goos = "darwin" - if got, want := strings.Join(cSharedExportArgs(ctx, pkgs), " "), "-Wl,-u,_Add -Wl,-u,_Zed"; got != want { + if got, want := strings.Join(cSharedExportArgs(ctx, summaries), " "), "-Wl,-u,_Add -Wl,-u,_Zed"; got != want { t.Fatalf("darwin cSharedExportArgs = %q, want %q", got, want) } ctx.buildConf.BuildMode = BuildModeExe - if got := cSharedExportArgs(ctx, pkgs); got != nil { + if got := cSharedExportArgs(ctx, summaries); got != nil { t.Fatalf("executable cSharedExportArgs = %v, want nil", got) } } diff --git a/internal/build/collect.go b/internal/build/collect.go index a27f83879f..8cefe1925e 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -328,6 +328,12 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if !cacheEnabled() { return false } + // C archive/shared builds still generate their public headers from live + // package declarations. Keep that compatibility path uncached until header + // declaration rendering is also represented by PackageSummary. + if c.buildConf != nil && (c.buildConf.BuildMode == BuildModeCArchive || c.buildConf.BuildMode == BuildModeCShared) { + return false + } // Main packages are intentionally not written to the build cache because // each executable's entry module is linked against the current main archive. @@ -371,6 +377,13 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { if err != nil { return false } + // Old manifests did not carry the linker-facing package summary. Treat + // those entries as misses rather than linking with incomplete ABI/metadata + // state. Released builds naturally invalidate caches through LLGo version; + // development builds safely repopulate the affected entries on demand. + if meta.Summary == nil { + return false + } // Use the .a archive directly for linking (no extraction needed) pkg.ArchiveFile = paths.Archive @@ -379,6 +392,7 @@ func (c *context) tryLoadFromCache(pkg *aPackage) bool { pkg.NeedPyInit = meta.NeedPyInit pkg.Meta = pkgMeta pkg.CacheHit = true + pkg.Summary = summaryFromMetadata(pkg, meta) return true } @@ -393,6 +407,7 @@ func parseManifestMetadata(content string) (*cacheArchiveMetadata, error) { meta.LinkArgs = append([]string(nil), data.Metadata.LinkArgs...) meta.NeedRt = data.Metadata.NeedRt meta.NeedPyInit = data.Metadata.NeedPyInit + meta.Summary = data.Metadata.Summary } return meta, nil } @@ -446,6 +461,7 @@ type cacheArchiveMetadata struct { LinkArgs []string NeedRt bool NeedPyInit bool + Summary *packageSummaryMetadata } // saveToCache saves a built package to cache. @@ -506,12 +522,9 @@ func (c *context) saveToCache(pkg *aPackage) error { LinkArgs: append([]string(nil), pkg.LinkArgs...), NeedRt: pkg.NeedRt, NeedPyInit: pkg.NeedPyInit, + Summary: pkg.Summary.metadata(), } - if len(meta.LinkArgs) == 0 && !meta.NeedRt && !meta.NeedPyInit { - data.Metadata = nil - } else { - data.Metadata = meta - } + data.Metadata = meta manifestWithMeta, err := buildManifestYAML(data) if err != nil { diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 0dd9d7ca4d..7ddb8b5816 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "strings" "testing" @@ -789,6 +790,81 @@ func TestTryLoadFromCache_ForceRebuild(t *testing.T) { } } +func TestTryLoadFromCacheRestoresPackageSummary(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "darwin", Goarch: "arm64"}, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + } + pkg := &aPackage{ + Package: &packages.Package{ID: "example.com/cached", PkgPath: "example.com/cached", Name: "cached"}, + Fingerprint: "summary-test", + Manifest: func() string { + m := newManifestBuilder() + m.env.Goos = "darwin" + m.pkg.PkgPath = "example.com/cached" + return m.Build() + }(), + NeedRt: true, + NeedPyInit: true, + LinkArgs: []string{"-lcached"}, + Summary: &PackageSummary{ + ID: "example.com/cached", + PkgPath: "example.com/cached", + LinkArgs: []string{"-lcached"}, + NeedRuntime: true, + NeedPyInit: true, + NeedAbiInit: 3, + MethodByIndex: []int{1}, + MethodByName: []string{"Method"}, + GlobalSymbols: []string{"example.com/cached.global"}, + FuncInfo: []funcInfoRecord{{symbol: "example.com/cached.fn", name: "Fn", file: "p.go", line: 7}}, + PCLineInfo: []pcLineRecord{{id: 9, symbol: "example.com/cached.fn", file: "p.go", line: 8}}, + FuncInfoStubs: []string{closureStubPrefix + "example.com/cached.fn"}, + CSharedExports: []string{"Cached"}, + }, + } + want := *pkg.Summary + obj, err := os.CreateTemp(td, "cached-*.o") + if err != nil { + t.Fatal(err) + } + if _, err := obj.WriteString("object"); err != nil { + t.Fatal(err) + } + if err := obj.Close(); err != nil { + t.Fatal(err) + } + pkg.ObjFiles = []string{obj.Name()} + if err := ctx.saveToCache(pkg); err != nil { + t.Fatalf("saveToCache: %v", err) + } + + pkg.ObjFiles = nil + pkg.ArchiveFile = "" + pkg.LinkArgs = nil + pkg.NeedRt = false + pkg.NeedPyInit = false + pkg.Summary = nil + if !ctx.tryLoadFromCache(pkg) { + t.Fatal("tryLoadFromCache = false, want summary cache hit") + } + if pkg.Summary == nil { + t.Fatal("cache hit did not restore package summary") + } + want.ArchiveFile = pkg.ArchiveFile + if !reflect.DeepEqual(pkg.Summary, &want) { + t.Fatalf("restored summary = %#v, want %#v", pkg.Summary, &want) + } +} + func TestSaveToCache_MainPackage(t *testing.T) { td := t.TempDir() oldFunc := cacheRootFunc diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index 242a8c7afa..901ec64f6c 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -37,9 +37,10 @@ type depEntry struct { // manifestMetadata stores metadata produced during build but not part of the fingerprint. type manifestMetadata struct { - LinkArgs []string `yaml:"link_args,omitempty"` - NeedRt bool `yaml:"need_rt,omitempty"` - NeedPyInit bool `yaml:"need_py_init,omitempty"` + LinkArgs []string `yaml:"link_args,omitempty"` + NeedRt bool `yaml:"need_rt,omitempty"` + NeedPyInit bool `yaml:"need_py_init,omitempty"` + Summary *packageSummaryMetadata `yaml:"summary,omitempty"` } // manifestData is the structured representation of manifest content. diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 30089f1c7a..7945ca289d 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -86,12 +86,16 @@ type funcInfoSymbolIndexRecord struct { } func collectFuncInfo(pkgs []Package) []funcInfoRecord { + return collectFuncInfoSummaries(summariesForPackages(pkgs)) +} + +func collectFuncInfoSummaries(summaries []*PackageSummary) []funcInfoRecord { seen := make(map[string]funcInfoRecord) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, rec := range readFuncInfo(pkg.LPkg.Module()) { + for _, rec := range summary.FuncInfo { if rec.symbol == "" { continue } @@ -114,13 +118,17 @@ func collectFuncInfo(pkgs []Package) []funcInfoRecord { } func collectPCLineInfo(pkgs []Package) []pcLineRecord { + return collectPCLineInfoSummaries(summariesForPackages(pkgs)) +} + +func collectPCLineInfoSummaries(summaries []*PackageSummary) []pcLineRecord { var out []pcLineRecord seen := make(map[uint64]none) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - for _, rec := range readPCLineInfo(pkg.LPkg.Module()) { + for _, rec := range summary.PCLineInfo { if rec.id == 0 || rec.symbol == "" { continue } @@ -144,6 +152,10 @@ func collectPCLineInfo(pkgs []Package) []pcLineRecord { } func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []funcInfoStubRecord { + return collectFuncInfoStubRecordsSummaries(summariesForPackages(pkgs), records) +} + +func collectFuncInfoStubRecordsSummaries(summaries []*PackageSummary, records []funcInfoRecord) []funcInfoStubRecord { if len(records) == 0 { return nil } @@ -154,23 +166,16 @@ func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []func } } seen := make(map[string]funcInfoStubRecord) - for _, pkg := range pkgs { - if pkg == nil || pkg.LPkg == nil { + for _, summary := range summaries { + if summary == nil { continue } - fn := pkg.LPkg.Module().FirstFunction() - for !fn.IsNil() { - if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { - fn = llvm.NextFunction(fn) - continue - } - name := fn.Name() + for _, name := range summary.FuncInfoStubs { if target, ok := strings.CutPrefix(name, closureStubPrefix); ok { if idx := recordBySymbol[target]; idx != 0 { seen[name] = funcInfoStubRecord{symbol: name, funcIndex: idx} } } - fn = llvm.NextFunction(fn) } } if len(seen) == 0 { diff --git a/internal/build/package_summary.go b/internal/build/package_summary.go new file mode 100644 index 0000000000..db13bed841 --- /dev/null +++ b/internal/build/package_summary.go @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + "strings" + + "github.com/xgo-dev/llvm" +) + +// PackageSummary is the immutable output a package contributes to the final +// link. It deliberately contains no LLVM values: a backend worker can produce +// it, emit its object, and then dispose its worker-local Program. +// +// C archive/shared header declarations still use LPkg for now. They have a +// separate type-rendering boundary and are kept on the serial compatibility +// path until that boundary is represented here as well. +type PackageSummary struct { + ID string + PkgPath string + + LinkArgs []string + ArchiveFile string + NeedRuntime bool + NeedPyInit bool + + NeedAbiInit int + MethodByIndex []int + MethodByName []string + GlobalSymbols []string + + FuncInfo []funcInfoRecord + PCLineInfo []pcLineRecord + FuncInfoStubs []string + CSharedExports []string +} + +// packageSummaryMetadata is the cache-safe representation of PackageSummary. +// Keep it separate from the in-memory type so the linker-facing structs can +// retain compact unexported record fields. +type packageSummaryMetadata struct { + NeedAbiInit int `yaml:"need_abi_init,omitempty"` + MethodByIndex []int `yaml:"method_by_index,omitempty"` + MethodByName []string `yaml:"method_by_name,omitempty"` + GlobalSymbols []string `yaml:"global_symbols,omitempty"` + + FuncInfo []funcInfoMetadata `yaml:"func_info,omitempty"` + PCLineInfo []pcLineMetadata `yaml:"pcline_info,omitempty"` + FuncInfoStubs []string `yaml:"func_info_stubs,omitempty"` + CSharedExports []string `yaml:"c_shared_exports,omitempty"` +} + +type funcInfoMetadata struct { + Symbol string `yaml:"symbol"` + Name string `yaml:"name,omitempty"` + File string `yaml:"file,omitempty"` + Line uint32 `yaml:"line,omitempty"` + Column uint32 `yaml:"column,omitempty"` +} + +type pcLineMetadata struct { + ID uint64 `yaml:"id"` + Symbol string `yaml:"symbol"` + File string `yaml:"file,omitempty"` + Line uint32 `yaml:"line,omitempty"` + Column uint32 `yaml:"column,omitempty"` +} + +func summarizePackage(pkg *aPackage) *PackageSummary { + if pkg == nil { + return nil + } + summary := &PackageSummary{ + LinkArgs: append([]string(nil), pkg.LinkArgs...), + ArchiveFile: pkg.ArchiveFile, + NeedRuntime: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + } + if pkg.Package != nil { + summary.ID = pkg.ID + summary.PkgPath = pkg.PkgPath + } + if pkg.LPkg == nil { + return summary + } + + lpkg := pkg.LPkg + if summary.PkgPath == "" { + summary.PkgPath = lpkg.Path() + } + summary.NeedAbiInit = lpkg.NeedAbiInit + for method := range lpkg.MethodByIndex { + summary.MethodByIndex = append(summary.MethodByIndex, method) + } + sort.Ints(summary.MethodByIndex) + for method := range lpkg.MethodByName { + summary.MethodByName = append(summary.MethodByName, method) + } + sort.Strings(summary.MethodByName) + + mod := lpkg.Module() + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if !global.IsDeclaration() { + summary.GlobalSymbols = append(summary.GlobalSymbols, global.Name()) + } + } + sort.Strings(summary.GlobalSymbols) + summary.FuncInfo = readFuncInfo(mod) + summary.PCLineInfo = readPCLineInfo(mod) + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { + continue + } + if _, ok := strings.CutPrefix(fn.Name(), closureStubPrefix); ok { + summary.FuncInfoStubs = append(summary.FuncInfoStubs, fn.Name()) + } + } + sort.Strings(summary.FuncInfoStubs) + for _, name := range lpkg.ExportFuncs() { + if name != "" { + summary.CSharedExports = append(summary.CSharedExports, name) + } + } + sort.Strings(summary.CSharedExports) + return summary +} + +func (s *PackageSummary) metadata() *packageSummaryMetadata { + if s == nil { + return nil + } + meta := &packageSummaryMetadata{ + NeedAbiInit: s.NeedAbiInit, + MethodByIndex: append([]int(nil), s.MethodByIndex...), + MethodByName: append([]string(nil), s.MethodByName...), + GlobalSymbols: append([]string(nil), s.GlobalSymbols...), + FuncInfoStubs: append([]string(nil), s.FuncInfoStubs...), + CSharedExports: append([]string(nil), s.CSharedExports...), + } + for _, rec := range s.FuncInfo { + meta.FuncInfo = append(meta.FuncInfo, funcInfoMetadata{ + Symbol: rec.symbol, + Name: rec.name, + File: rec.file, + Line: rec.line, + Column: rec.column, + }) + } + for _, rec := range s.PCLineInfo { + meta.PCLineInfo = append(meta.PCLineInfo, pcLineMetadata{ + ID: rec.id, + Symbol: rec.symbol, + File: rec.file, + Line: rec.line, + Column: rec.column, + }) + } + return meta +} + +func summaryFromMetadata(pkg *aPackage, meta *cacheArchiveMetadata) *PackageSummary { + if pkg == nil || pkg.Package == nil || meta == nil || meta.Summary == nil { + return nil + } + summary := &PackageSummary{ + ID: pkg.ID, + PkgPath: pkg.PkgPath, + LinkArgs: append([]string(nil), pkg.LinkArgs...), + ArchiveFile: pkg.ArchiveFile, + NeedRuntime: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + NeedAbiInit: meta.Summary.NeedAbiInit, + MethodByIndex: append([]int(nil), meta.Summary.MethodByIndex...), + MethodByName: append([]string(nil), meta.Summary.MethodByName...), + GlobalSymbols: append([]string(nil), meta.Summary.GlobalSymbols...), + FuncInfoStubs: append([]string(nil), meta.Summary.FuncInfoStubs...), + CSharedExports: append([]string(nil), meta.Summary.CSharedExports...), + } + for _, rec := range meta.Summary.FuncInfo { + summary.FuncInfo = append(summary.FuncInfo, funcInfoRecord{ + symbol: rec.Symbol, + name: rec.Name, + file: rec.File, + line: rec.Line, + column: rec.Column, + }) + } + for _, rec := range meta.Summary.PCLineInfo { + summary.PCLineInfo = append(summary.PCLineInfo, pcLineRecord{ + id: rec.ID, + symbol: rec.Symbol, + file: rec.File, + line: rec.Line, + column: rec.Column, + }) + } + return summary +} + +func summariesForPackages(pkgs []Package) []*PackageSummary { + summaries := make([]*PackageSummary, 0, len(pkgs)) + for _, pkg := range pkgs { + if pkg == nil { + continue + } + summary := pkg.Summary + if summary == nil { + summary = summarizePackage(pkg) + } + if summary != nil { + summaries = append(summaries, summary) + } + } + return summaries +} diff --git a/internal/build/package_summary_test.go b/internal/build/package_summary_test.go new file mode 100644 index 0000000000..87a60f42ea --- /dev/null +++ b/internal/build/package_summary_test.go @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/types" + "reflect" + "testing" + + "github.com/xgo-dev/llvm" + + "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" +) + +func TestPackageSummaryCapturesLinkerFacts(t *testing.T) { + prog := llssa.NewProgram(nil) + defer prog.Dispose() + lpkg := prog.NewPackage("example.com/p", "p") + lpkg.NeedAbiInit = 3 + lpkg.RecordReflectMethodByIndex(4) + lpkg.RecordReflectMethodByIndex(1) + lpkg.RecordReflectMethodByName("B") + lpkg.RecordReflectMethodByName("A") + lpkg.SetExport("example.com/p.Export", "Export") + lpkg.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "p.go", 17, 2) + lpkg.EmitPCLineInfo(42, "example.com/p.live", "p.go", 18, 3) + lpkg.NewFunc(closureStubPrefix+"example.com/p.live", llssa.NoArgsNoRet, llssa.InGo).MakeBody(1).Return() + + i32 := lpkg.Module().Context().Int32Type() + defined := llvm.AddGlobal(lpkg.Module(), i32, "example.com/p.defined") + defined.SetInitializer(llvm.ConstInt(i32, 1, false)) + llvm.AddGlobal(lpkg.Module(), i32, "example.com/p.declared") + + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/p", + PkgPath: "example.com/p", + Types: types.NewPackage("example.com/p", "p"), + }, + LPkg: lpkg, + NeedRt: true, + NeedPyInit: true, + LinkArgs: []string{"-lp"}, + ArchiveFile: "p.a", + } + summary := summarizePackage(pkg) + if got, want := summary.MethodByIndex, []int{1, 4}; !reflect.DeepEqual(got, want) { + t.Fatalf("MethodByIndex = %v, want %v", got, want) + } + if got, want := summary.MethodByName, []string{"A", "B"}; !reflect.DeepEqual(got, want) { + t.Fatalf("MethodByName = %v, want %v", got, want) + } + if got, want := summary.GlobalSymbols, []string{"example.com/p.defined"}; !reflect.DeepEqual(got, want) { + t.Fatalf("GlobalSymbols = %v, want %v", got, want) + } + if got, want := summary.FuncInfoStubs, []string{closureStubPrefix + "example.com/p.live"}; !reflect.DeepEqual(got, want) { + t.Fatalf("FuncInfoStubs = %v, want %v", got, want) + } + if got := collectFuncInfoSummaries([]*PackageSummary{summary}); len(got) != 1 || got[0].symbol != "example.com/p.live" { + t.Fatalf("func info from summary = %+v, want live record", got) + } + if got := linkedPackageGlobals([]*PackageSummary{summary}); len(got) != 1 { + t.Fatalf("globals from summary = %#v, want one defined global", got) + } + + loadedPkg := &aPackage{Package: pkg.Package, LinkArgs: []string{"-lp"}, ArchiveFile: "p.a", NeedRt: true, NeedPyInit: true} + loaded := summaryFromMetadata(loadedPkg, &cacheArchiveMetadata{Summary: summary.metadata()}) + if !reflect.DeepEqual(loaded, summary) { + t.Fatalf("cache summary round trip = %#v, want %#v", loaded, summary) + } +} From df50e8039a978dc44c57673597d173bf56ecdfe3 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 12:54:05 +0800 Subject: [PATCH 11/23] build: summarize skipped linker packages --- internal/build/build.go | 7 +++++++ internal/build/package_build_test.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 36d3196696..bd7074e75f 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -934,6 +934,10 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s ctx.builtMu.Unlock() if spec.isDeclOnly() { pkg.ExportFile = "" + // Declaration-only packages still occur in the link graph through a + // different packages.Package instance. Publish an empty summary so the + // linker can treat that occurrence like every other skipped package. + aPkg.Summary = summarizePackage(aPkg) return true, ctx.collectFingerprint(aPkg) } if spec.isLinkOnly() && !spec.hasSource() { @@ -941,6 +945,9 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } + // Link-only packages do not create an LLVM module, but their linker + // arguments are still consumed by linkMainPkg through PackageSummary. + aPkg.Summary = summarizePackage(aPkg) return true, ctx.collectFingerprint(aPkg) } if err := ctx.collectFingerprint(aPkg); err != nil { diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 7b3e26ace1..ff38f14e41 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -136,7 +136,7 @@ func TestPreflightFingerprintsSkippedPackage(t *testing.T) { if err != nil { t.Fatal(err) } - if !skip || pkg.Fingerprint == "" || pkg.Manifest == "" { - t.Fatalf("skipped package was not fingerprinted: skip=%v fingerprint=%q manifest=%q", skip, pkg.Fingerprint, pkg.Manifest) + if !skip || pkg.Fingerprint == "" || pkg.Manifest == "" || pkg.Summary == nil { + t.Fatalf("skipped package was not fully prepared: skip=%v fingerprint=%q manifest=%q summary=%#v", skip, pkg.Fingerprint, pkg.Manifest, pkg.Summary) } } From 717d135334aeca037fe45bba310ec9679080e0bd Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 12:56:38 +0800 Subject: [PATCH 12/23] build: skip unbuilt runtime packages at link --- internal/build/build.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/build/build.go b/internal/build/build.go index bd7074e75f..2fa76b15d6 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1241,6 +1241,13 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa aPkg = ctx.pkgByID[p.ID] } if p.ExportFile != "" && aPkg != nil { // skip packages that only contain declarations + // Runtime packages are built on demand. For a target that does not + // require the runtime, preserve the historical behavior of omitting + // those unbuilt packages from the link rather than requiring a + // linker summary that no backend ever produced. + if aPkg.Summary == nil && isRuntimePkg(aPkg.PkgPath) { + return + } linkedPkgs[p.ID] = true linkedOrder = append(linkedOrder, aPkg) } From 6094d5ce7534cd2c49e83b08e27a268e1df2c2bd Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 13:16:19 +0800 Subject: [PATCH 13/23] build: persist empty package summaries in cache --- internal/build/collect.go | 3 +++ internal/build/collect_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/build/collect.go b/internal/build/collect.go index 8cefe1925e..c33a3fe3c1 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -517,6 +517,9 @@ func (c *context) saveToCache(pkg *aPackage) error { if err != nil { return fmt.Errorf("decode manifest: %w", err) } + if pkg.Summary == nil { + pkg.Summary = summarizePackage(pkg) + } meta := &manifestMetadata{ LinkArgs: append([]string(nil), pkg.LinkArgs...), diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 7ddb8b5816..84df370fe0 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -1017,8 +1017,8 @@ func TestSaveToCache_Success(t *testing.T) { if data.Env.Goos != "darwin" { t.Errorf("manifest should contain original env content") } - if data.Metadata != nil { - t.Errorf("metadata should be empty when no link args/runtime flags") + if data.Metadata == nil || data.Metadata.Summary == nil { + t.Errorf("metadata should preserve an empty linker summary") } // Check archive exists From 823d2c0cbdb0b00791016ea0cd1242f9f3d08aba Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 26 Jul 2026 16:01:03 +0800 Subject: [PATCH 14/23] build: adapt package summary test to reflect API --- internal/build/package_summary_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/build/package_summary_test.go b/internal/build/package_summary_test.go index 87a60f42ea..e634529096 100644 --- a/internal/build/package_summary_test.go +++ b/internal/build/package_summary_test.go @@ -32,10 +32,10 @@ func TestPackageSummaryCapturesLinkerFacts(t *testing.T) { defer prog.Dispose() lpkg := prog.NewPackage("example.com/p", "p") lpkg.NeedAbiInit = 3 - lpkg.RecordReflectMethodByIndex(4) - lpkg.RecordReflectMethodByIndex(1) - lpkg.RecordReflectMethodByName("B") - lpkg.RecordReflectMethodByName("A") + lpkg.RecordReflectMethodByIndex("example.com/p.Method", 4) + lpkg.RecordReflectMethodByIndex("example.com/p.Method", 1) + lpkg.RecordReflectMethodByName("example.com/p.MethodByName", "B") + lpkg.RecordReflectMethodByName("example.com/p.MethodByName", "A") lpkg.SetExport("example.com/p.Export", "Export") lpkg.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "p.go", 17, 2) lpkg.EmitPCLineInfo(42, "example.com/p.live", "p.go", 18, 3) From 0f6621817db7f707fb53e42b86b539e3398c046f Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 26 Jul 2026 18:42:08 +0800 Subject: [PATCH 15/23] test: cover cache meta bypass with package summary --- internal/build/collect_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 84df370fe0..227bc29f31 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -1192,6 +1192,10 @@ func TestTryLoadFromCacheIgnoresMetaWhenPackageMetaDisabled(t *testing.T) { m := newManifestBuilder() m.env.Goos = "darwin" m.pkg.PkgPath = "example.com/nometa" + // Cache entries always carry a linker summary, even when package metadata + // is disabled. The malformed sidecar below must therefore be ignored rather + // than making this otherwise valid cache entry miss. + m.meta = &manifestMetadata{Summary: &packageSummaryMetadata{}} if err := writeManifest(paths.Manifest, m.Build()); err != nil { t.Fatal(err) } From 6ddacf54b6d08b287a0d13058ee7a88d83791721 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 08:07:01 +0800 Subject: [PATCH 16/23] build: template isolated backend programs --- internal/build/build.go | 110 ++++++++++++++++++++++++++++++----- internal/build/build_test.go | 20 +++++++ 2 files changed, 115 insertions(+), 15 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 2fa76b15d6..5e1d5edb79 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -395,7 +395,14 @@ func Do(args []string, conf *Config) ([]Package, error) { OptLevel: conf.OptLevel, } - prog := llssa.NewProgram(target) + funcInfo := conf.Mode != ModeGen && conf.PCLNMode != PCLNNone + // Site records are inline-asm fragments inside function bodies. Darwin + // DWARF builds avoid them because they disturb LLDB lexical scopes; Linux + // still needs them because its restricted dynamic symbol table cannot + // reconstruct every Go entry PC through dlsym. External mode always needs + // final-PC sites for sidecar construction. + backendTemplate := newBackendProgramTemplate(target, conf, funcInfo, shouldEnablePCLNSites(conf, funcInfo, emitDebugInfo)) + prog := backendTemplate.newProgram() if conf.Mode != ModeGen { // ModeGen callers (llgen and the golden suites) read LPkg.String() // after Do returns and dispose the program themselves; every other @@ -405,20 +412,6 @@ func Do(args []string, conf *Config) ([]Package, error) { // harness) otherwise accumulate every compile's C++-side memory. defer prog.Dispose() } - prog.EnableGoGlobalDCE(conf.goGlobalDCEEnabled()) - prog.EnableDeadcodeDrop(conf.deadcodeDropEnabled()) - if conf.PthreadStackSize > 0 { - prog.SetPthreadStackSize(uint64(conf.PthreadStackSize)) - } - prog.EnableLTOPluginMarkers(conf.LTOPlugin.Enabled()) - funcInfo := conf.Mode != ModeGen && conf.PCLNMode != PCLNNone - prog.EnableFuncInfoMetadata(funcInfo) - // Site records are inline-asm fragments inside function bodies. Darwin - // DWARF builds avoid them because they disturb LLDB lexical scopes; Linux - // still needs them because its restricted dynamic symbol table cannot - // reconstruct every Go entry PC through dlsym. External mode always needs - // final-PC sites for sidecar construction. - prog.EnableFuncInfoSites(shouldEnablePCLNSites(conf, funcInfo, emitDebugInfo)) sizes := func(sizes types.Sizes, compiler, arch string) types.Sizes { if arch == "wasm" { sizes = &types.StdSizes{WordSize: 4, MaxAlign: 4} @@ -509,6 +502,12 @@ func Do(args []string, conf *Config) ([]Package, error) { if !IsOptimizeEnabled() { buildMode |= ssa.NaiveForm } + backendTemplate.runtimePackage = func() *types.Package { return altPkgs[0].Types } + backendTemplate.pythonPackage = func() *types.Package { return dedup.Check(llssa.PkgPython).Types } + backendTemplate.runtimeLinknames = altPkgs + backendTemplate.llvmTarget = export.LLVMTarget + backendTemplate.targetABI = export.TargetABI + backendTemplate.cabiOptimize = cabiOptimize progSSA := ssa.NewProgram(initial[0].Fset, buildMode) patches := make(cl.Patches, len(altPkgPaths)) @@ -526,6 +525,7 @@ func Do(args []string, conf *Config) ([]Package, error) { buildConf: conf, crossCompile: export, cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + backend: backendTemplate, sfilesCache: make(map[string][]string), } defer ctx.closePackageMetas() @@ -768,6 +768,7 @@ type context struct { crossCompile crosscompile.Export cTransformer *cabi.Transformer + backend backendProgramTemplate testFail bool @@ -804,6 +805,85 @@ func (c *context) closePackageMetas() { } } +// backendProgramTemplate contains all state a backend Program needs without +// sharing an LLVM context. The current serial path keeps one Program in +// context.prog; the next stage will create one backendSession per worker from +// this template. +type backendProgramTemplate struct { + target *llssa.Target + goGlobalDCE bool + deadcodeDrop bool + pthreadStackSize int64 + ltoPluginMarkers bool + funcInfoMetadata bool + funcInfoSites bool + runtimePackage func() *types.Package + pythonPackage func() *types.Package + runtimeLinknames []*packages.Package + llvmTarget string + targetABI string + abiMode cabi.Mode + cabiOptimize bool +} + +type backendSession struct { + prog llssa.Program + transformer *cabi.Transformer +} + +func newBackendProgramTemplate(target *llssa.Target, conf *Config, funcInfoMetadata, funcInfoSites bool) backendProgramTemplate { + var targetCopy *llssa.Target + if target != nil { + copy := *target + targetCopy = © + } + return backendProgramTemplate{ + target: targetCopy, + goGlobalDCE: conf.goGlobalDCEEnabled(), + deadcodeDrop: conf.deadcodeDropEnabled(), + pthreadStackSize: conf.PthreadStackSize, + ltoPluginMarkers: conf.LTOPlugin.Enabled(), + funcInfoMetadata: funcInfoMetadata, + funcInfoSites: funcInfoSites, + abiMode: conf.AbiMode, + } +} + +func (t backendProgramTemplate) newProgram() llssa.Program { + var target *llssa.Target + if t.target != nil { + copy := *t.target + target = © + } + prog := llssa.NewProgram(target) + prog.EnableGoGlobalDCE(t.goGlobalDCE) + prog.EnableDeadcodeDrop(t.deadcodeDrop) + if t.pthreadStackSize > 0 { + prog.SetPthreadStackSize(uint64(t.pthreadStackSize)) + } + prog.EnableLTOPluginMarkers(t.ltoPluginMarkers) + prog.EnableFuncInfoMetadata(t.funcInfoMetadata) + prog.EnableFuncInfoSites(t.funcInfoSites) + if t.runtimePackage != nil { + prog.SetRuntime(t.runtimePackage) + } + if t.pythonPackage != nil { + prog.SetPython(t.pythonPackage) + } + if len(t.runtimeLinknames) != 0 { + preCollectRuntimeLinknames(prog, t.runtimeLinknames) + } + return prog +} + +func (t backendProgramTemplate) newSession() backendSession { + prog := t.newProgram() + return backendSession{ + prog: prog, + transformer: cabi.NewTransformer(prog, t.llvmTarget, t.targetABI, t.abiMode, t.cabiOptimize), + } +} + func (c *context) compiler() *clang.Cmd { config := clang.NewConfig( c.crossCompile.CC, diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 105c4cec8a..7a974cd580 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -94,6 +94,26 @@ func TestNeedsLinuxNoPIE(t *testing.T) { } } +func TestBackendProgramTemplateCreatesIsolatedSessions(t *testing.T) { + conf := &Config{Goos: "linux", Goarch: "amd64"} + template := newBackendProgramTemplate(&llssa.Target{GOOS: conf.Goos, GOARCH: conf.Goarch}, conf, true, true) + first := template.newSession() + defer first.prog.Dispose() + second := template.newSession() + defer second.prog.Dispose() + if first.transformer == nil || second.transformer == nil { + t.Fatal("backend session missing C ABI transformer") + } + if !first.prog.FuncInfoMetadataEnabled() || !first.prog.FuncInfoSitesEnabled() { + t.Fatal("backend template did not preserve funcinfo configuration") + } + firstModule := first.prog.NewPackage("example.com/first", "first").Module() + secondModule := second.prog.NewPackage("example.com/second", "second").Module() + if firstModule.Context().C == secondModule.Context().C { + t.Fatal("backend sessions share an LLVM context") + } +} + func TestNeedsLinuxExportDynamic(t *testing.T) { t.Setenv(llgoFuncInfo, "") ctx := &context{buildConf: &Config{Goos: "linux"}} From aa7648519e530828195797e116facc6807fa356c Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 26 Jul 2026 18:48:45 +0800 Subject: [PATCH 17/23] runtime: synchronize FuncForPC cache --- .../lib/runtime/pprof_runtime_stub_llgo.go | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index c74b6d1421..b80970b6ea 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -5,6 +5,7 @@ package runtime import ( "unsafe" + psync "github.com/goplus/llgo/runtime/internal/clite/pthread/sync" llrt "github.com/goplus/llgo/runtime/internal/runtime" ) @@ -99,11 +100,19 @@ type funcForPCCacheEntry struct { var funcForPCCache [funcForPCCacheSets][funcForPCCacheWays]funcForPCCacheEntry var funcForPCCacheNext [funcForPCCacheSets]uint8 var funcForPCLast funcForPCCacheEntry +var funcForPCCacheOnce psync.Once +var funcForPCCacheMu psync.Mutex -func FuncForPC(pc uintptr) *Func { - // External metadata must be installed before consulting either cache: - // caching a pre-load miss would otherwise survive a successful load. - ensureRuntimePCLN() +func lockFuncForPCCache() { + funcForPCCacheOnce.Do(func() { + funcForPCCacheMu.Init(nil) + }) + funcForPCCacheMu.Lock() +} + +func cachedFuncForPC(pc uintptr) *Func { + lockFuncForPCCache() + defer funcForPCCacheMu.Unlock() if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { return fn } @@ -114,6 +123,16 @@ func FuncForPC(pc uintptr) *Func { return fn } } + return nil +} + +func FuncForPC(pc uintptr) *Func { + // External metadata must be installed before consulting either cache: + // caching a pre-load miss would otherwise survive a successful load. + ensureRuntimePCLN() + if fn := cachedFuncForPC(pc); fn != nil { + return fn + } return funcForPCSlow(pc) } @@ -226,15 +245,9 @@ func newFuncForPC(pc uintptr, sym pcSymbol) *Func { // symbolized, going through the FuncForPC cache so repeated CallersFrames // walks over the same PCs stop allocating a Func per frame. func frameFuncForPC(pc uintptr, sym pcSymbol, name string) *Func { - if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { + if fn := cachedFuncForPC(pc); fn != nil { return fn } - set := &funcForPCCache[funcForPCCacheIndex(pc)] - for i := 0; i < funcForPCCacheWays; i++ { - if fn := set[i].fn; fn != nil && set[i].pc == pc { - return fn - } - } fn := &Func{ entry: sym.entry, name: name, @@ -252,6 +265,8 @@ func frameFuncForPC(pc uintptr, sym pcSymbol, name string) *Func { } func cacheFuncForPC(pc uintptr, fn *Func) { + lockFuncForPCCache() + defer funcForPCCacheMu.Unlock() setIndex := funcForPCCacheIndex(pc) set := &funcForPCCache[setIndex] for i := 0; i < funcForPCCacheWays; i++ { From a8b02eac66efaea0cd7b627c97af302ab6c40cad Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 26 Jul 2026 19:53:32 +0800 Subject: [PATCH 18/23] runtime: avoid pthread initialization in FuncForPC cache --- .../lib/runtime/pprof_runtime_stub_llgo.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index b80970b6ea..2d0a5b8aa4 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -5,7 +5,7 @@ package runtime import ( "unsafe" - psync "github.com/goplus/llgo/runtime/internal/clite/pthread/sync" + latomic "github.com/goplus/llgo/runtime/internal/lib/sync/atomic" llrt "github.com/goplus/llgo/runtime/internal/runtime" ) @@ -100,19 +100,20 @@ type funcForPCCacheEntry struct { var funcForPCCache [funcForPCCacheSets][funcForPCCacheWays]funcForPCCacheEntry var funcForPCCacheNext [funcForPCCacheSets]uint8 var funcForPCLast funcForPCCacheEntry -var funcForPCCacheOnce psync.Once -var funcForPCCacheMu psync.Mutex +var funcForPCCacheLock uint32 func lockFuncForPCCache() { - funcForPCCacheOnce.Do(func() { - funcForPCCacheMu.Init(nil) - }) - funcForPCCacheMu.Lock() + for !latomic.CompareAndSwapUint32(&funcForPCCacheLock, 0, 1) { + } +} + +func unlockFuncForPCCache() { + latomic.StoreUint32(&funcForPCCacheLock, 0) } func cachedFuncForPC(pc uintptr) *Func { lockFuncForPCCache() - defer funcForPCCacheMu.Unlock() + defer unlockFuncForPCCache() if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { return fn } @@ -266,7 +267,7 @@ func frameFuncForPC(pc uintptr, sym pcSymbol, name string) *Func { func cacheFuncForPC(pc uintptr, fn *Func) { lockFuncForPCCache() - defer funcForPCCacheMu.Unlock() + defer unlockFuncForPCCache() setIndex := funcForPCCacheIndex(pc) set := &funcForPCCache[setIndex] for i := 0; i < funcForPCCacheWays; i++ { From fc8be13d2a768453525ae66713009e514ab8cd90 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 26 Jul 2026 21:28:45 +0800 Subject: [PATCH 19/23] runtime: avoid defer in FuncForPC cache lock --- runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 2d0a5b8aa4..451b65bb03 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -113,17 +113,19 @@ func unlockFuncForPCCache() { func cachedFuncForPC(pc uintptr) *Func { lockFuncForPCCache() - defer unlockFuncForPCCache() if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { + unlockFuncForPCCache() return fn } set := &funcForPCCache[funcForPCCacheIndex(pc)] for i := 0; i < funcForPCCacheWays; i++ { if fn := set[i].fn; fn != nil && set[i].pc == pc { funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} + unlockFuncForPCCache() return fn } } + unlockFuncForPCCache() return nil } @@ -267,13 +269,13 @@ func frameFuncForPC(pc uintptr, sym pcSymbol, name string) *Func { func cacheFuncForPC(pc uintptr, fn *Func) { lockFuncForPCCache() - defer unlockFuncForPCCache() setIndex := funcForPCCacheIndex(pc) set := &funcForPCCache[setIndex] for i := 0; i < funcForPCCacheWays; i++ { if set[i].fn == nil || set[i].pc == pc { set[i] = funcForPCCacheEntry{pc: pc, fn: fn} funcForPCLast = set[i] + unlockFuncForPCCache() return } } @@ -281,6 +283,7 @@ func cacheFuncForPC(pc uintptr, fn *Func) { funcForPCCacheNext[setIndex] = way + 1 set[way] = funcForPCCacheEntry{pc: pc, fn: fn} funcForPCLast = set[way] + unlockFuncForPCCache() } func funcForPCCacheIndex(pc uintptr) uintptr { From b6b54107f02fd4d5c17ef23c4c92447788ccc45e Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 08:20:04 +0800 Subject: [PATCH 20/23] cl: precompute caller tracking for backend workers --- cl/caller_tracking_precompute_test.go | 66 +++++++++++++++++++++++++++ cl/instr.go | 47 +++++++++++++++++++ internal/build/build.go | 3 ++ 3 files changed, 116 insertions(+) create mode 100644 cl/caller_tracking_precompute_test.go diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go new file mode 100644 index 0000000000..bab5bad861 --- /dev/null +++ b/cl/caller_tracking_precompute_test.go @@ -0,0 +1,66 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "sync" + "testing" + + gossa "golang.org/x/tools/go/ssa" +) + +func TestCallerTrackingPrecomputeFreezesConcurrentReads(t *testing.T) { + dep, root := buildCallerFrameSSAProgram(t, + "example.com/dep", `package dep +import "runtime" +func Where() { runtime.Caller(0) } +`, + "example.com/root", `package root +import "example.com/dep" +func Logs() { dep.Where() } +`) + tracking := NewCallerTracking() + tracking.Precompute([]*gossa.Package{root}) + if !tracking.frozen { + t.Fatal("CallerTracking was not frozen after precomputation") + } + if !runtimeCallerBaseSet(tracking, dep)[dep.Func("Where")] { + t.Fatal("precomputed base set lost runtime caller function") + } + if !runtimeCallerFuncSet(tracking, root)[root.Func("Logs")] { + t.Fatal("precomputed extended set lost cross-package caller") + } + + var wg sync.WaitGroup + errs := make(chan struct{}, 32) + for range 32 { + wg.Add(1) + go func() { + defer wg.Done() + if !runtimeCallerBaseSet(tracking, dep)[dep.Func("Where")] || !runtimeCallerFuncSet(tracking, root)[root.Func("Logs")] { + errs <- struct{}{} + } + }() + } + wg.Wait() + close(errs) + if len(errs) != 0 { + t.Fatal("concurrent read lost precomputed caller tracking data") + } +} diff --git a/cl/instr.go b/cl/instr.go index 678c4afc6d..378f391977 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -25,6 +25,7 @@ import ( "log" "os" "regexp" + "sort" "strings" "golang.org/x/tools/go/ssa" @@ -924,6 +925,9 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function if set, ok := c.extended[pkg]; ok { return set } + if c.frozen { + return nil + } base := runtimeCallerBaseSet(c, pkg) out := make(map[*ssa.Function]bool, len(base)) for fn := range base { @@ -983,6 +987,7 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function type CallerTracking struct { base map[*ssa.Package]map[*ssa.Function]bool extended map[*ssa.Package]map[*ssa.Function]bool + frozen bool } // NewCallerTracking creates the caller-tracking memoization for one @@ -994,6 +999,45 @@ func NewCallerTracking() *CallerTracking { } } +// Precompute resolves caller-tracking state for every package in the supplied +// SSA programs, then freezes the memoization for concurrent reads. Drivers +// must call it after SSA Build and before compiling packages on independent +// LLVM contexts. A frozen tracker deliberately returns no data for an unknown +// package instead of mutating its maps from a worker. +func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { + if c == nil || c.frozen { + return + } + all := make(map[*ssa.Package]bool) + for _, pkg := range pkgs { + if pkg == nil { + continue + } + all[pkg] = true + if pkg.Prog != nil { + for _, programPkg := range pkg.Prog.AllPackages() { + if programPkg != nil { + all[programPkg] = true + } + } + } + } + ordered := make([]*ssa.Package, 0, len(all)) + for pkg := range all { + ordered = append(ordered, pkg) + } + sort.Slice(ordered, func(i, j int) bool { + return llssa.PathOf(ordered[i].Pkg) < llssa.PathOf(ordered[j].Pkg) + }) + for _, pkg := range ordered { + runtimeCallerBaseSet(c, pkg) + } + for _, pkg := range ordered { + runtimeCallerFuncSet(c, pkg) + } + c.frozen = true +} + func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { if fn == nil || fn.Parent() != nil { return false @@ -1012,6 +1056,9 @@ func runtimeCallerBaseSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function if set, ok := c.base[pkg]; ok { return set } + if c.frozen { + return nil + } set := computeRuntimeCallerBaseSet(pkg) c.base[pkg] = set return set diff --git a/internal/build/build.go b/internal/build/build.go index 5e1d5edb79..cf38646851 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -544,6 +544,9 @@ func Do(args []string, conf *Config) ([]Package, error) { return nil, err } buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) + // CallerTracking owns plain maps. Resolve every possible query while the + // SSA graph is still serial, so worker-local backend sessions only read it. + ctx.callerTracking.Precompute(ctx.progSSA.AllPackages()) allPkgs := append([]*aPackage{}, pkgs...) allPkgs = append(allPkgs, depPkgs...) From 74d9ea0608af419f9d5507c9ece59ca5d082fd5a Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 15:48:05 +0800 Subject: [PATCH 21/23] cl: cover frozen caller tracking misses --- cl/caller_tracking_precompute_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go index bab5bad861..58663c57fa 100644 --- a/cl/caller_tracking_precompute_test.go +++ b/cl/caller_tracking_precompute_test.go @@ -63,4 +63,18 @@ func Logs() { dep.Where() } if len(errs) != 0 { t.Fatal("concurrent read lost precomputed caller tracking data") } + + // A worker can encounter a package outside the precomputed program. Once + // frozen, that must be a read-only miss rather than lazy map mutation. + delete(tracking.base, dep) + if got := runtimeCallerBaseSet(tracking, dep); got != nil { + t.Fatalf("frozen base lookup for unknown package = %v, want nil", got) + } + delete(tracking.extended, root) + if got := runtimeCallerFuncSet(tracking, root); got != nil { + t.Fatalf("frozen extended lookup for unknown package = %v, want nil", got) + } + + // Repeated setup is intentionally a no-op after the tracker is frozen. + tracking.Precompute(nil) } From fdd16ba033e052093cb254f2782cc309836b1201 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 08:50:04 +0800 Subject: [PATCH 22/23] build: run isolated LLVM backends by package --- internal/build/build.go | 180 ++++++++++++++++++++++++++------ internal/build/build_test.go | 22 ++++ internal/build/collect.go | 4 +- internal/build/package_build.go | 79 ++++++++++++++ internal/build/plan9asm.go | 67 ++++++++---- 5 files changed, 298 insertions(+), 54 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index cf38646851..3b61162633 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -412,18 +412,33 @@ func Do(args []string, conf *Config) ([]Package, error) { // harness) otherwise accumulate every compile's C++-side memory. defer prog.Dispose() } + var backendTypeSizes types.Sizes + var backendTypeSizesMu sync.Mutex sizes := func(sizes types.Sizes, compiler, arch string) types.Sizes { if arch == "wasm" { sizes = &types.StdSizes{WordSize: 4, MaxAlign: 4} } + backendTypeSizesMu.Lock() + if backendTypeSizes == nil { + backendTypeSizes = sizes + } + backendTypeSizesMu.Unlock() return prog.TypeSizes(sizes) } dedup := packages.NewDeduper() + var backendSyntaxInputs []backendSyntaxInput + var backendSyntaxInputsMu sync.Mutex dedup.SetPreload(func(pkg *types.Package, files []*ast.File) { if llruntime.SkipToBuild(pkg.Path()) { return } cl.ParsePkgSyntax(prog, pkg, files) + backendSyntaxInputsMu.Lock() + backendSyntaxInputs = append(backendSyntaxInputs, backendSyntaxInput{ + pkg: pkg, + files: slices.Clone(files), + }) + backendSyntaxInputsMu.Unlock() }) if patterns == nil { @@ -505,6 +520,8 @@ func Do(args []string, conf *Config) ([]Package, error) { backendTemplate.runtimePackage = func() *types.Package { return altPkgs[0].Types } backendTemplate.pythonPackage = func() *types.Package { return dedup.Check(llssa.PkgPython).Types } backendTemplate.runtimeLinknames = altPkgs + backendTemplate.typeSizes = backendTypeSizes + backendTemplate.syntaxInputs = backendSyntaxInputs backendTemplate.llvmTarget = export.LLVMTarget backendTemplate.targetABI = export.TargetABI backendTemplate.cabiOptimize = cabiOptimize @@ -526,7 +543,8 @@ func Do(args []string, conf *Config) ([]Package, error) { crossCompile: export, cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), backend: backendTemplate, - sfilesCache: make(map[string][]string), + sfiles: &sfilesState{cache: make(map[string][]string)}, + plan9asm: &plan9asmState{}, } defer ctx.closePackageMetas() ctx.initializePackageBuildState() @@ -543,6 +561,7 @@ func Do(args []string, conf *Config) ([]Package, error) { if err != nil { return nil, err } + appendPatchedSyntaxInputs(ctx) buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) // CallerTracking owns plain maps. Resolve every possible query while the // SSA graph is still serial, so worker-local backend sessions only read it. @@ -782,14 +801,9 @@ type context struct { llvmVersionReady bool llvmVersionMu sync.Mutex - // go list derived file lists (SFiles, etc.) - sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths - sfilesMu sync.Mutex - - // plan9asm package policy parsed from env. - plan9asmOnce sync.Once - plan9asmMode plan9asmPkgsEnvMode - plan9asmPkgs map[string]bool + // Shared worker-safe state that does not belong to one LLVM Program. + sfiles *sfilesState + plan9asm *plan9asmState // pclnExternal is populated while generating the synthetic main module // and completed with final linked PCs by the post-link externalizer. @@ -809,17 +823,18 @@ func (c *context) closePackageMetas() { } // backendProgramTemplate contains all state a backend Program needs without -// sharing an LLVM context. The current serial path keeps one Program in -// context.prog; the next stage will create one backendSession per worker from -// this template. +// sharing an LLVM context. The coordinator keeps its Program for compatibility +// paths; each executable package worker creates its own backendSession. type backendProgramTemplate struct { target *llssa.Target + typeSizes types.Sizes goGlobalDCE bool deadcodeDrop bool pthreadStackSize int64 ltoPluginMarkers bool funcInfoMetadata bool funcInfoSites bool + syntaxInputs []backendSyntaxInput runtimePackage func() *types.Package pythonPackage func() *types.Package runtimeLinknames []*packages.Package @@ -829,6 +844,15 @@ type backendProgramTemplate struct { cabiOptimize bool } +// backendSyntaxInput is the package-level compiler metadata loaded by +// go/packages before compilation begins. A serial build accumulates this state +// in one Program as packages are compiled; each isolated backend Program must +// start from the complete, immutable input set instead. +type backendSyntaxInput struct { + pkg *types.Package + files []*ast.File +} + type backendSession struct { prog llssa.Program transformer *cabi.Transformer @@ -859,6 +883,9 @@ func (t backendProgramTemplate) newProgram() llssa.Program { target = © } prog := llssa.NewProgram(target) + if t.typeSizes != nil { + prog.TypeSizes(t.typeSizes) + } prog.EnableGoGlobalDCE(t.goGlobalDCE) prog.EnableDeadcodeDrop(t.deadcodeDrop) if t.pthreadStackSize > 0 { @@ -873,6 +900,10 @@ func (t backendProgramTemplate) newProgram() llssa.Program { if t.pythonPackage != nil { prog.SetPython(t.pythonPackage) } + for _, input := range t.syntaxInputs { + cl.ParsePkgSyntax(prog, input.pkg, input.files) + cl.PreCollectLinknames(prog, input.pkg.Path(), input.files) + } if len(t.runtimeLinknames) != 0 { preCollectRuntimeLinknames(prog, t.runtimeLinknames) } @@ -887,6 +918,29 @@ func (t backendProgramTemplate) newSession() backendSession { } } +// appendPatchedSyntaxInputs replays alternate-runtime directives under the +// original package path. Patches replace an SSA package's types while keeping +// its original identity; their directives are normally accumulated when that +// package is compiled in the shared Program. Worker Programs need the same +// remapping before compiling any consumer of the patched package. +func appendPatchedSyntaxInputs(ctx *context) { + paths := make([]string, 0, len(ctx.patches)) + for pkgPath := range ctx.patches { + paths = append(paths, pkgPath) + } + slices.Sort(paths) + for _, pkgPath := range paths { + alt := ctx.dedup.Check(altPkgPathPrefix + pkgPath) + if alt == nil || len(alt.Syntax) == 0 { + continue + } + ctx.backend.syntaxInputs = append(ctx.backend.syntaxInputs, backendSyntaxInput{ + pkg: types.NewPackage(pkgPath, ""), + files: slices.Clone(alt.Syntax), + }) + } +} + func (c *context) compiler() *clang.Cmd { config := clang.NewConfig( c.crossCompile.CC, @@ -955,33 +1009,26 @@ func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, er if err != nil { return nil, err } - // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []packageBuildSpec - var normalPkgs []packageBuildSpec - for _, spec := range plan.specs { - if spec.runtime { - runtimePkgs = append(runtimePkgs, spec) - } else { - normalPkgs = append(normalPkgs, spec) - } - } - var needRuntime, needPyInit bool // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, spec := range normalPkgs { - result, err := buildPreflightedPackage(ctx, preflights[spec.pkg.ID], verbose) + for _, level := range plan.levels { + level = packageBuildLevelWithoutRuntime(level) + results, err := buildPreflightedPackageLevel(ctx, level, preflights, verbose) if err != nil { return nil, err } - needRuntime = needRuntime || result.needRuntime - needPyInit = needPyInit || result.needPyInit + for _, result := range results { + needRuntime = needRuntime || result.needRuntime + needPyInit = needPyInit || result.needPyInit + } } // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, spec := range runtimePkgs { - if _, err := buildPreflightedPackage(ctx, preflights[spec.pkg.ID], verbose); err != nil { + for _, level := range plan.levels { + level = packageBuildLevelRuntimeOnly(level) + if _, err := buildPreflightedPackageLevel(ctx, level, preflights, verbose); err != nil { return nil, err } } @@ -1048,10 +1095,26 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s } // executePackageBuild creates the frontend module and, on a cache miss, runs -// the serial LLVM backend. The backend remains deliberately serial until it -// owns an isolated LLVM context. +// the LLVM backend. Eligible executable builds use a fresh backend Program for +// each package, so packageBuildPlan can schedule ready transactions under -p. func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) error { aPkg := spec.pkg + if !aPkg.CacheHit && ctx.canUseIsolatedBackend() { + defer func() { aPkg.LPkg = nil }() + return ctx.withIsolatedBackend(func(task *context) error { + if err := buildPkg(task, aPkg, verbose); err != nil { + return err + } + if spec.needsRuntimeSignals() { + aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) + } + // Capture all linker-facing state before the worker Program is + // disposed. finalizePackageBuild will add the archive path after + // normalizing the emitted objects. + aPkg.Summary = summarizePackage(aPkg) + return nil + }) + } if err := buildPkg(ctx, aPkg, verbose); err != nil { return err } @@ -1074,15 +1137,64 @@ func finalizePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (pa if spec.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, spec.kindParam) } - // Snapshot every linker-facing LLVM fact before cache publication. The - // summary is the hand-off point to a future worker-local Program. - aPkg.Summary = summarizePackage(aPkg) + // Snapshot every linker-facing LLVM fact before cache publication. An + // isolated backend has already captured LLVM-owned state before disposing + // its Program; only update values produced by finalization here. + if aPkg.Summary == nil { + aPkg.Summary = summarizePackage(aPkg) + } else { + aPkg.Summary.ArchiveFile = aPkg.ArchiveFile + aPkg.Summary.LinkArgs = append(aPkg.Summary.LinkArgs[:0], aPkg.LinkArgs...) + } if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } return packageBuildResultFor(spec), nil } +// canUseIsolatedBackend excludes modes that expose live LLVM modules or still +// need them after compilation (notably C headers). It is the compatibility +// gate for package-DAG LLVM workers. +func (ctx *context) canUseIsolatedBackend() bool { + return ctx.mode == ModeBuild && + ctx.buildConf.BuildMode == BuildModeExe && + ctx.buildConf.ModuleHook == nil +} + +func (ctx *context) withIsolatedBackend(fn func(task *context) error) error { + session := ctx.backend.newSession() + defer session.prog.Dispose() + return fn(ctx.newBackendTask(session)) +} + +// newBackendTask creates a context with no shared LLVM state. Every field +// copied here is immutable after package preflight, or points to state with +// its own synchronization. Do not add the coordinator's mutex-bearing cache +// or build bookkeeping fields here: backend workers must not need them. +func (ctx *context) newBackendTask(session backendSession) *context { + return &context{ + env: ctx.env, + conf: ctx.conf, + progSSA: ctx.progSSA, + prog: session.prog, + dedup: ctx.dedup, + patches: ctx.patches, + callerTracking: ctx.callerTracking, + initial: ctx.initial, + pkgs: ctx.pkgs, + pkgByID: ctx.pkgByID, + mode: ctx.mode, + output: ctx.output, + passOpt: ctx.passOpt, + buildConf: ctx.buildConf, + crossCompile: ctx.crossCompile, + cTransformer: session.transformer, + backend: ctx.backend, + sfiles: ctx.sfiles, + plan9asm: ctx.plan9asm, + } +} + func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { // need to be linked with external library // format: ';' separated alternative link methods. e.g. diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 7a974cd580..c13e87ba90 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -10,6 +10,7 @@ import ( "go/ast" "go/parser" "go/token" + "go/types" "io" "os" "os/exec" @@ -114,6 +115,27 @@ func TestBackendProgramTemplateCreatesIsolatedSessions(t *testing.T) { } } +func TestBackendProgramTemplateReplaysPackageSyntax(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "p.go", `package p +//go:linkname target host_target +func target() +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + conf := &Config{Goos: "linux", Goarch: "amd64"} + template := newBackendProgramTemplate(&llssa.Target{GOOS: conf.Goos, GOARCH: conf.Goarch}, conf, false, false) + template.syntaxInputs = []backendSyntaxInput{{ + pkg: types.NewPackage("example.com/p", "p"), + files: []*ast.File{file}, + }} + session := template.newSession() + defer session.prog.Dispose() + if got, ok := session.prog.Linkname("example.com/p.target"); !ok || got != "host_target" { + t.Fatalf("replayed linkname = (%q, %v), want (host_target, true)", got, ok) + } +} + func TestNeedsLinuxExportDynamic(t *testing.T) { t.Setenv(llgoFuncInfo, "") ctx := &context{buildConf: &Config{Goos: "linux"}} diff --git a/internal/build/collect.go b/internal/build/collect.go index c33a3fe3c1..c148f2c2e5 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -210,8 +210,8 @@ func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage, fin // has to exist before parallel preflight begins. LLVM version discovery stays // lazy and is synchronized by getLLVMVersion. func (c *context) initializePackageBuildState() { - if c.sfilesCache == nil { - c.sfilesCache = make(map[string][]string) + if c.sfiles == nil { + c.sfiles = &sfilesState{cache: make(map[string][]string)} } if cacheEnabled() { c.cacheManager = newCacheManager() diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 4ee5f925b8..6a2b2a8e63 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -82,6 +82,85 @@ func preflightPackageBuilds(ctx *context, plan *packageBuildPlan, verbose bool) return preflights, nil } +func packageBuildLevelWithoutRuntime(level []packageBuildSpec) []packageBuildSpec { + return packageBuildLevelFilter(level, false) +} + +func packageBuildLevelRuntimeOnly(level []packageBuildSpec) []packageBuildSpec { + return packageBuildLevelFilter(level, true) +} + +func packageBuildLevelFilter(level []packageBuildSpec, runtime bool) []packageBuildSpec { + filtered := make([]packageBuildSpec, 0, len(level)) + for _, spec := range level { + if spec.runtime == runtime { + filtered = append(filtered, spec) + } + } + return filtered +} + +// buildPreflightedPackageLevel runs ready packages under the same -p bound as +// preflight. Only the isolated executable backend takes this path; all other +// modes retain the compatibility serial order because they expose live LLVM +// modules or use post-compile declarations. +func buildPreflightedPackageLevel(ctx *context, level []packageBuildSpec, preflights map[string]packagePreflight, verbose bool) ([]packageBuildResult, error) { + if len(level) == 0 { + return nil, nil + } + if !ctx.canUseIsolatedBackend() || len(level) == 1 { + results := make([]packageBuildResult, 0, len(level)) + for _, spec := range level { + result, err := buildPreflightedPackage(ctx, preflights[spec.pkg.ID], verbose) + if err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil + } + + type result struct { + index int + value packageBuildResult + err error + } + workers := min(ctx.buildConf.parallelism(), len(level)) + jobs := make(chan int) + completed := make(chan result, len(level)) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + preflight := preflights[level[index].pkg.ID] + value, err := buildPreflightedPackage(ctx, preflight, verbose) + completed <- result{index: index, value: value, err: err} + } + }() + } + for index := range level { + jobs <- index + } + close(jobs) + wg.Wait() + close(completed) + + results := make([]packageBuildResult, len(level)) + var firstErr error + for result := range completed { + if result.err != nil && firstErr == nil { + firstErr = result.err + } + results[result.index] = result.value + } + if firstErr != nil { + return nil, firstErr + } + return results, nil +} + func newPackageBuildSpec(pkg *aPackage) packageBuildSpec { kind, kindParam := cl.PkgKindOf(pkg.Types) return packageBuildSpec{ diff --git a/internal/build/plan9asm.go b/internal/build/plan9asm.go index 88b2cf1595..fefff2de50 100644 --- a/internal/build/plan9asm.go +++ b/internal/build/plan9asm.go @@ -18,6 +18,11 @@ import ( gllvm "github.com/xgo-dev/llvm" ) +// xgo-dev/plan9asm parses generated IR through llvm.GlobalContext. Keep that +// auxiliary translation path serialized until the dependency accepts an +// explicit LLVM context; package backend Programs remain independent. +var plan9asmGlobalContextMu sync.Mutex + // compilePkgSFiles translates Go/Plan9 assembly files selected by `go list -json` // for this package/target into LLVM IR, compiles them to .o, and returns the // object files for linking. @@ -67,8 +72,10 @@ func compilePkgSFiles(ctx *context, aPkg *aPackage, pkg *packages.Package, verbo if shouldSkipDarwinDynimportTrampolineAsm(skipDarwinDynimportTrampolines, sfile, src) { continue } + plan9asmGlobalContextMu.Lock() tr, err := llplan9asm.TranslateSourceModuleForPkg(pkg, sfile, src, ctx.buildConf.Goos, ctx.buildConf.Goarch) if err != nil { + plan9asmGlobalContextMu.Unlock() // Some stdlib .s files are comment-only placeholders (e.g. internal/cpu/cpu.s). // Skip those silently. if strings.Contains(err.Error(), "no TEXT directive found") { @@ -88,6 +95,7 @@ func compilePkgSFiles(ctx *context, aPkg *aPackage, pkg *packages.Package, verbo } ll := mod.String() mod.Dispose() + plan9asmGlobalContextMu.Unlock() baseName := aPkg.ExportFile + filepath.Base(sfile) // used for stable debug output paths tmpPrefix := "plan9asm-" + filepath.Base(sfile) + "-" @@ -191,6 +199,17 @@ type plan9asmPkgsEnv struct { pkgs map[string]bool } +type plan9asmState struct { + once sync.Once + mode plan9asmPkgsEnvMode + pkgs map[string]bool +} + +type sfilesState struct { + mu sync.Mutex + cache map[string][]string +} + func parsePlan9AsmPkgsEnv(raw string) plan9asmPkgsEnv { v := strings.TrimSpace(raw) switch { @@ -261,7 +280,9 @@ func plan9asmSigsForPkg(ctx *context, pkgPath string) (map[string]struct{}, erro if err != nil { return nil, fmt.Errorf("%s: read %s: %w", pkg.PkgPath, sfile, err) } + plan9asmGlobalContextMu.Lock() tr, err := llplan9asm.TranslateSourceForPkg(pkg, sfile, src, ctx.buildConf.Goos, ctx.buildConf.Goarch) + plan9asmGlobalContextMu.Unlock() if err != nil { if strings.Contains(err.Error(), "no TEXT directive found") { continue @@ -306,27 +327,31 @@ func cabiSkipFuncsForPlan9Asm(ctx *context, pkgPath string, mod gllvm.Module) [] } func (ctx *context) plan9asmEnabled(pkgPath string) bool { - ctx.plan9asmOnce.Do(func() { + if ctx.plan9asm == nil { + ctx.plan9asm = &plan9asmState{} + } + state := ctx.plan9asm + state.once.Do(func() { cfg := parsePlan9AsmPkgsEnv(Plan9ASMPkgs()) - ctx.plan9asmMode = cfg.mode + state.mode = cfg.mode switch cfg.mode { case plan9asmEnvSelected: - ctx.plan9asmPkgs = make(map[string]bool, len(cfg.pkgs)) + state.pkgs = make(map[string]bool, len(cfg.pkgs)) for p := range cfg.pkgs { - ctx.plan9asmPkgs[p] = true + state.pkgs[p] = true } default: - ctx.plan9asmPkgs = make(map[string]bool) + state.pkgs = make(map[string]bool) } }) - switch ctx.plan9asmMode { + switch state.mode { case plan9asmEnvAll: return true case plan9asmEnvNone: return false case plan9asmEnvSelected: - return ctx.plan9asmPkgs[pkgPath] + return state.pkgs[pkgPath] case plan9asmEnvDefaults: return plan9asmEnabledByDefault(ctx.buildConf, pkgPath) default: @@ -457,25 +482,31 @@ func pkgSFiles(ctx *context, pkg *packages.Package) ([]string, error) { } func getCachedSFiles(ctx *context, pkgID string) ([]string, bool) { - ctx.sfilesMu.Lock() - defer ctx.sfilesMu.Unlock() - if ctx.sfilesCache == nil { - ctx.sfilesCache = make(map[string][]string) + if ctx.sfiles == nil { + ctx.sfiles = &sfilesState{} } - paths, ok := ctx.sfilesCache[pkgID] + ctx.sfiles.mu.Lock() + defer ctx.sfiles.mu.Unlock() + if ctx.sfiles.cache == nil { + ctx.sfiles.cache = make(map[string][]string) + } + paths, ok := ctx.sfiles.cache[pkgID] return paths, ok } func cacheSFiles(ctx *context, pkgID string, paths []string) []string { - ctx.sfilesMu.Lock() - defer ctx.sfilesMu.Unlock() - if ctx.sfilesCache == nil { - ctx.sfilesCache = make(map[string][]string) + if ctx.sfiles == nil { + ctx.sfiles = &sfilesState{} + } + ctx.sfiles.mu.Lock() + defer ctx.sfiles.mu.Unlock() + if ctx.sfiles.cache == nil { + ctx.sfiles.cache = make(map[string][]string) } - if cached, ok := ctx.sfilesCache[pkgID]; ok { + if cached, ok := ctx.sfiles.cache[pkgID]; ok { return cached } - ctx.sfilesCache[pkgID] = paths + ctx.sfiles.cache[pkgID] = paths return paths } From d13de585a334b16da9a330302eb7396b4276eb19 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 25 Jul 2026 13:15:33 +0800 Subject: [PATCH 23/23] build: isolate cached package backends --- internal/build/build.go | 15 +++++++++++++-- internal/build/collect.go | 3 +++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 3b61162633..8d238bdcd7 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1099,10 +1099,10 @@ func preflightPackageBuild(ctx *context, spec packageBuildSpec, verbose bool) (s // each package, so packageBuildPlan can schedule ready transactions under -p. func executePackageBuild(ctx *context, spec packageBuildSpec, verbose bool) error { aPkg := spec.pkg - if !aPkg.CacheHit && ctx.canUseIsolatedBackend() { + if ctx.canUseIsolatedBackend() { defer func() { aPkg.LPkg = nil }() return ctx.withIsolatedBackend(func(task *context) error { - if err := buildPkg(task, aPkg, verbose); err != nil { + if err := buildIsolatedPkg(task, aPkg, verbose); err != nil { return err } if spec.needsRuntimeSignals() { @@ -1167,6 +1167,17 @@ func (ctx *context) withIsolatedBackend(fn func(task *context) error) error { return fn(ctx.newBackendTask(session)) } +// buildIsolatedPkg builds a package entirely in its worker-local LLVM session. +// Module construction, transformation, optimization, and object emission can +// therefore proceed concurrently with other packages. +func buildIsolatedPkg(ctx *context, aPkg *aPackage, verbose bool) error { + externs, err := preparePackageModule(ctx, aPkg, verbose) + if err != nil || aPkg.CacheHit || aPkg.LPkg == nil { + return err + } + return compilePackageModule(ctx, aPkg, externs, verbose) +} + // newBackendTask creates a context with no shared LLVM state. Every field // copied here is immutable after package preflight, or points to state with // its own synchronization. Do not add the coordinator's mutex-bearing cache diff --git a/internal/build/collect.go b/internal/build/collect.go index c148f2c2e5..193c9039b0 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -517,6 +517,9 @@ func (c *context) saveToCache(pkg *aPackage) error { if err != nil { return fmt.Errorf("decode manifest: %w", err) } + // Every cache entry consumed by the package linker needs a summary, even + // when the package contributes no special link metadata. This also keeps + // direct saveToCache callers compatible with isolated backend workers. if pkg.Summary == nil { pkg.Summary = summarizePackage(pkg) }