From 8f520a5a9086078a4001f1555663a5b021ea959a Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Sun, 12 Jul 2026 16:11:15 +0800 Subject: [PATCH 1/3] feat(formula): discover matrix keys with SSA tracker --- AGENTS.md | 63 +++++ doc/matrix-options-tracker.md | 53 ++++ internal/formula/formula.go | 12 +- internal/formula/formula_test.go | 26 ++ internal/formula/probe.go | 67 +++++ .../formula/testdata/formula/matrix_llar.gox | 20 ++ .../testdata/formula/trackcases_llar.gox | 77 ++++++ .../testdata/formula/trackfilter_llar.gox | 8 + .../testdata/formula/trackpanic_llar.gox | 12 + internal/formula/tracker.go | 249 ++++++++++++++++++ internal/formula/tracker_test.go | 78 ++++++ 11 files changed, 663 insertions(+), 2 deletions(-) create mode 100644 doc/matrix-options-tracker.md create mode 100644 internal/formula/probe.go create mode 100644 internal/formula/testdata/formula/matrix_llar.gox create mode 100644 internal/formula/testdata/formula/trackcases_llar.gox create mode 100644 internal/formula/testdata/formula/trackfilter_llar.gox create mode 100644 internal/formula/testdata/formula/trackpanic_llar.gox create mode 100644 internal/formula/tracker.go create mode 100644 internal/formula/tracker_test.go diff --git a/AGENTS.md b/AGENTS.md index 219b0c45..efd68cba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,69 @@ func main() { 3. **Error Handling**: If a file doesn't match any registered classfile pattern (wrong suffix), `BuildFile` returns "undefined" errors for DSL functions +### Classfile `var` Fields + +In an XGo classfile, the first top-level `var (...)` declaration is the class +fields declaration. These names compile into fields on the generated struct, +not package-level variables. If the fields have initializers, XGo generates an +`XGo_Init()` method and calls it before the classfile body runs. + +Source file `hello_llar.gox`: +```gox +var ( + seen map[string]bool = {} +) + +func mark(key string) { + seen[key] = true +} + +id "DaveGamble/cJSON" +fromVer "v1.0.0" + +onRequire (proj, deps) => { + mark "zlib" +} +``` + +Generated Go code: +```go +package main + +import "github.com/goplus/llar/formula" + +type hello struct { + formula.ModuleF + seen map[string]bool +} + +func (this *hello) mark(key string) { + this.seen[key] = true +} + +func (this *hello) MainEntry() { + this.XGo_Init() + this.Id("DaveGamble/cJSON") + this.FromVer("v1.0.0") + this.OnRequire(func(proj *formula.Project, deps *formula.ModuleDeps) { + this.mark("zlib") + }) +} + +func (this *hello) Main() { + formula.Gopt_ModuleF_Main(this) +} + +func (this *hello) XGo_Init() *hello { + this.seen = map[string]bool{} + return this +} + +func main() { + new(hello).Main() +} +``` + ## Build & Test ```bash diff --git a/doc/matrix-options-tracker.md b/doc/matrix-options-tracker.md new file mode 100644 index 00000000..61e39f8b --- /dev/null +++ b/doc/matrix-options-tracker.md @@ -0,0 +1,53 @@ +# Proposal: Discover Formula Matrix with SSA Instrumentation + +## Summary + +LLAR discovers matrix keys by observing formula execution. Formula code remains +unchanged; matrix reads are instrumented in Go SSA after ixgo has built the +package and before ixgo translates it for execution. + +## Pipeline + +```text +_llar.gox source + -> generated Go source + -> x/tools SSA and built-in optimization + -> tracker call insertion + -> ixgo interpreter +``` + +The tracker does not rewrite XGo AST or modify ixgo. + +## Instrumentation + +XGo lowers a matrix read into a method call followed by `ssa.Lookup`: + +```text +target.options[key] + -> matrixTarget.Options() + -> map lookup +``` + +The tracker inserts no-result calls at two points: + +1. After `matrixTarget.Options()` or `matrixTarget.Require()`, register the + returned map as options or require. +2. Before each `ssa.Lookup` on `map[string][]string`, record the key only when + the map was registered in step 1. + +Map identity follows normal assignments and function arguments, so aliases and +helpers do not require data-flow reconstruction. + +## Probe + +`LoadFS` executes the formula hooks with empty probe inputs, then copies the +observed require and options keys into `Formula.Matrix`. Values are filled later +from the selected matrix. Tracker calls are disabled after the probe and do not +affect normal builds. + +## Boundaries + +- Instrumentation runs after all x/tools SSA builder passes. +- Inserted calls do not produce values or change control flow. +- Only code represented in the ixgo SSA program can be instrumented. +- Additional probe rounds and candidate generation are separate concerns. diff --git a/internal/formula/formula.go b/internal/formula/formula.go index 1d8879b7..542d983d 100644 --- a/internal/formula/formula.go +++ b/internal/formula/formula.go @@ -28,6 +28,7 @@ type Formula struct { // the method declaration of ModuleF in formula/classfile.go ModPath string FromVer string + Matrix formula.Matrix OnRequire func(proj *formula.Project, deps *formula.ModuleDeps) OnBuild func(ctx *formula.Context, proj *formula.Project, out *formula.BuildResult) OnTest func(ctx *formula.Context, proj *formula.Project, out *formula.TestResult) @@ -94,6 +95,8 @@ func loadFS(fs fs.ReadFileFS, path string) (*Formula, error) { if err != nil { return nil, err } + tr := newTracker() + tracked := tr.track(ctx, pkgs) // Create a new interpreter for the loaded package interp, err := ctx.NewInterp(pkgs) @@ -132,7 +135,7 @@ func loadFS(fs fs.ReadFileFS, path string) (*Formula, error) { val.Interface().(interface{ Main() }).Main() // Extract the populated fields from the struct and return the Formula - return &Formula{ + loaded := &Formula{ structElem: class, ModPath: valueOf(class, "modPath").(string), FromVer: valueOf(class, "modFromVer").(string), @@ -140,7 +143,12 @@ func loadFS(fs fs.ReadFileFS, path string) (*Formula, error) { OnTest: valueOf(class, "fOnTest").(func(*formula.Context, *formula.Project, *formula.TestResult)), OnRequire: valueOf(class, "fOnRequire").(func(*formula.Project, *formula.ModuleDeps)), Filter: valueOf(class, "fFilter").(func() bool), - }, nil + } + if tracked { + probeFormula(loaded) + } + loaded.Matrix = tr.matrix() + return loaded, nil } // LoadFS loads a formula from a filesystem interface. diff --git a/internal/formula/formula_test.go b/internal/formula/formula_test.go index 49220414..6c1839e4 100644 --- a/internal/formula/formula_test.go +++ b/internal/formula/formula_test.go @@ -7,6 +7,7 @@ package formula import ( "io/fs" "os" + "reflect" "testing" formulapkg "github.com/goplus/llar/formula" @@ -79,6 +80,14 @@ func TestLoadFS_TargetSurface(t *testing.T) { if !f.Filter() { t.Fatal("Filter() = false, want true") } + wantRequire := map[string][]string{"os": nil} + if !reflect.DeepEqual(f.Matrix.Require, wantRequire) { + t.Fatalf("Matrix.Require = %#v, want %#v", f.Matrix.Require, wantRequire) + } + wantOptions := map[string][]string{"debug": nil, "zlib": nil} + if !reflect.DeepEqual(f.Matrix.Options, wantOptions) { + t.Fatalf("Matrix.Options = %#v, want %#v", f.Matrix.Options, wantOptions) + } var deps formulapkg.ModuleDeps f.OnRequire(&formulapkg.Project{}, &deps) @@ -89,6 +98,23 @@ func TestLoadFS_TargetSurface(t *testing.T) { f.OnBuild(&formulapkg.Context{}, &formulapkg.Project{}, &formulapkg.BuildResult{}) } +func TestLoadFSProbesMatrixKeys(t *testing.T) { + fsys := os.DirFS("testdata/formula").(fs.ReadFileFS) + f, err := LoadFS(fsys, "matrix_llar.gox") + if err != nil { + t.Fatalf("LoadFS failed: %v", err) + } + + wantRequire := map[string][]string{"os": nil} + if !reflect.DeepEqual(f.Matrix.Require, wantRequire) { + t.Fatalf("Matrix.Require = %#v, want %#v", f.Matrix.Require, wantRequire) + } + wantOptions := map[string][]string{"debug": nil, "ssl": nil} + if !reflect.DeepEqual(f.Matrix.Options, wantOptions) { + t.Fatalf("Matrix.Options = %#v, want %#v", f.Matrix.Options, wantOptions) + } +} + func TestFormula_SetStdout(t *testing.T) { formula, err := loadFS(os.DirFS("testdata/formula").(fs.ReadFileFS), "hello_llar.gox") if err != nil { diff --git a/internal/formula/probe.go b/internal/formula/probe.go new file mode 100644 index 00000000..3424f177 --- /dev/null +++ b/internal/formula/probe.go @@ -0,0 +1,67 @@ +// Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package formula + +import ( + "io/fs" + + classfile "github.com/goplus/llar/formula" + "github.com/goplus/llar/mod/module" +) + +type probeFS struct{} + +func (probeFS) Open(string) (fs.File, error) { + return nil, fs.ErrNotExist +} + +func (probeFS) ReadFile(string) ([]byte, error) { + return nil, fs.ErrNotExist +} + +func probeFormula(f *Formula) { + originalTarget := valueOf(f.structElem, "target").(classfile.Matrix) + // Probe with empty maps because discovery does not require a valid matrix. + // A formula may read several independent options while configuring a build: + // + // if has(target.options["zlib"], "ON") { ... } + // shared := has(target.options["shared"], "ON") + // debug := has(target.options["debug"], "ON") + // + // Each lookup returns an empty value, but the SSA tracker still records zlib, + // shared, and debug before the formula eventually succeeds or fails. The maps + // must be non-nil because their runtime identities also let the tracker follow + // aliases and helper arguments. + fakeTarget := classfile.Matrix{ + Require: make(map[string][]string), + Options: make(map[string][]string), + } + setValue(f.structElem, "target", fakeTarget) + defer setValue(f.structElem, "target", originalTarget) + + project := &classfile.Project{SourceFS: probeFS{}} + if f.OnRequire != nil { + var deps classfile.ModuleDeps + safeProbeCall(func() { + f.OnRequire(project, &deps) + }) + } + if f.OnBuild != nil { + ctx := classfile.NewContext("", "", "", func(string, module.Version) (string, error) { + return "", nil + }) + var out classfile.BuildResult + safeProbeCall(func() { + f.OnBuild(ctx, project, &out) + }) + } +} + +func safeProbeCall(call func()) { + defer func() { + _ = recover() + }() + call() +} diff --git a/internal/formula/testdata/formula/matrix_llar.gox b/internal/formula/testdata/formula/matrix_llar.gox new file mode 100644 index 00000000..f37f4b9d --- /dev/null +++ b/internal/formula/testdata/formula/matrix_llar.gox @@ -0,0 +1,20 @@ +func lookup(values map[string][]string, key string) []string { + return values[key] +} + +id "test/matrix" + +fromVer "v1.0.0" + +onRequire (proj, deps) => { + require := target.require + if len(lookup(require, "os")) == 0 { + options := target.options + _ = lookup(options, "ssl") + } +} + +onBuild (ctx, proj, out) => { + options := target.options + _ = len(lookup(options, "debug")) +} diff --git a/internal/formula/testdata/formula/trackcases_llar.gox b/internal/formula/testdata/formula/trackcases_llar.gox new file mode 100644 index 00000000..20a9ab79 --- /dev/null +++ b/internal/formula/testdata/formula/trackcases_llar.gox @@ -0,0 +1,77 @@ +type namedOptions map[string][]string + +type aliasedOptions = map[string][]string + +type matrixHolder struct { + values map[string][]string +} + +func matrixLookup(values map[string][]string, key string) []string { + return values[key] +} + +func matrixPassthrough(values map[string][]string) map[string][]string { + return values +} + +func dynamicMatrixKey() string { + return "dynamic" +} + +id "test/track-cases" + +fromVer "v1.0.0" + +onRequire (proj, deps) => { + _ = target.require["direct-require"] + + require := target.require + _ = matrixLookup(require, "helper-require") + _ = require["same"] + + unrelated := map[string][]string{"ignored-require": []string{"value"}} + _ = matrixLookup(unrelated, "ignored-require") +} + +onBuild (ctx, proj, out) => { + _ = target.options["direct-option"] + + options := target.options + alias := options + _ = alias["alias"] + _ = alias["alias"] + _ = matrixLookup(options, "helper-option") + _ = matrixLookup(matrixPassthrough(options), "returned") + + named := namedOptions(options) + _ = named["named"] + aliased := aliasedOptions(options) + _ = aliased["type-alias"] + + optionsPtr := &options + _ = (*optionsPtr)["pointer"] + + key := dynamicMatrixKey() + _ = options[key] + + _, ok := options["comma-ok"] + _ = ok + + holder := matrixHolder{values: options} + _ = holder.values["struct"] + + var boxed any = options + fromInterface := boxed.(map[string][]string) + _ = fromInterface["interface"] + + func() { + _ = options["closure"] + }() + + _ = options["same"] + + unrelated := map[string][]string{"ignored-option": []string{"value"}} + _ = matrixLookup(unrelated, "ignored-option") + var nilMap map[string][]string + _ = nilMap["ignored-nil"] +} diff --git a/internal/formula/testdata/formula/trackfilter_llar.gox b/internal/formula/testdata/formula/trackfilter_llar.gox new file mode 100644 index 00000000..1779346e --- /dev/null +++ b/internal/formula/testdata/formula/trackfilter_llar.gox @@ -0,0 +1,8 @@ +id "test/track-filter" + +fromVer "v1.0.0" + +filter => { + _ = target.options["filter-only"] + return false +} diff --git a/internal/formula/testdata/formula/trackpanic_llar.gox b/internal/formula/testdata/formula/trackpanic_llar.gox new file mode 100644 index 00000000..7215e1ce --- /dev/null +++ b/internal/formula/testdata/formula/trackpanic_llar.gox @@ -0,0 +1,12 @@ +id "test/track-panic" + +fromVer "v1.0.0" + +onRequire (proj, deps) => { + _ = target.options["before-panic"] + panic("stop require probe") +} + +onBuild (ctx, proj, out) => { + _ = target.options["after-panic"] +} diff --git a/internal/formula/tracker.go b/internal/formula/tracker.go new file mode 100644 index 00000000..eda450f3 --- /dev/null +++ b/internal/formula/tracker.go @@ -0,0 +1,249 @@ +// Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package formula + +import ( + "go/token" + "go/types" + "reflect" + "sync" + "unsafe" + + "github.com/goplus/ixgo" + classfile "github.com/goplus/llar/formula" + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" +) + +const ( + optionsTrackerFunc = "__internalOptionsTracker" + requireTrackerFunc = "__internalRequireTracker" + lookupTrackerFunc = "__internalLookupTracker" + formulaPackagePath = "github.com/goplus/llar/formula" +) + +type matrixKind uint8 + +const ( + unknownMatrix matrixKind = iota + requireMatrix + optionsMatrix +) + +var matrixMapType = types.NewMap(types.Typ[types.String], types.NewSlice(types.Typ[types.String])) + +type trackedMap struct { + kind matrixKind + // Keep the map alive so its runtime identity cannot be reused during probe. + values map[string][]string +} + +// tracker discovers matrix reads by instrumenting the completed Go SSA program. +// A target accessor registers the returned map, and each map lookup reports its +// map and key before the original lookup executes. For example: +// +// options := target.Options() // register options by map identity +// lookup(options, "shared") // report the lookup inside lookup +// +// Map identity survives assignment and argument passing, so lookups in helpers +// remain associated with Options or Require. Lookups on maps that were never +// returned by a target accessor are ignored at runtime. +type tracker struct { + mu sync.Mutex + + active bool + maps map[uintptr]trackedMap + require map[string]struct{} + options map[string]struct{} +} + +func newTracker() *tracker { + return &tracker{ + active: true, + maps: make(map[uintptr]trackedMap), + require: make(map[string]struct{}), + options: make(map[string]struct{}), + } +} + +// track instruments the completed SSA program before ixgo translates it. +func (t *tracker) track(ctx *ixgo.Context, pkg *ssa.Package) bool { + functions := ssautil.AllFunctions(pkg.Prog) + + valuesParam := types.NewVar(token.NoPos, nil, "values", matrixMapType) + registerSignature := types.NewSignatureType( + nil, nil, nil, + types.NewTuple(valuesParam), + types.NewTuple(), + false, + ) + keyParam := types.NewVar(token.NoPos, nil, "key", types.Typ[types.String]) + lookupSignature := types.NewSignatureType( + nil, nil, nil, + types.NewTuple(valuesParam, keyParam), + types.NewTuple(), + false, + ) + optionsTracker := pkg.Prog.NewFunction(optionsTrackerFunc, registerSignature, "llar matrix tracker") + requireTracker := pkg.Prog.NewFunction(requireTrackerFunc, registerSignature, "llar matrix tracker") + lookupTracker := pkg.Prog.NewFunction(lookupTrackerFunc, lookupSignature, "llar matrix tracker") + + ctx.RegisterExternal(optionsTracker.String(), func(values map[string][]string) { + t.trackMap(optionsMatrix, values) + }) + ctx.RegisterExternal(requireTracker.String(), func(values map[string][]string) { + t.trackMap(requireMatrix, values) + }) + ctx.RegisterExternal(lookupTracker.String(), t.trackLookup) + + tracked := false + for fn := range functions { + if fn.Blocks == nil { + continue + } + for _, block := range fn.Blocks { + instrs := make([]ssa.Instruction, 0, len(block.Instrs)) + for _, instr := range block.Instrs { + // Observe every matching lookup. trackLookup filters unrelated maps + // using the identities registered by Options or Require. + if lookup, ok := instr.(*ssa.Lookup); ok && isMatrixMap(lookup.X.Type()) { + instrs = append(instrs, trackerCall(block, lookupTracker, lookup.X, lookup.Index)) + } + instrs = append(instrs, instr) + if call, ok := instr.(*ssa.Call); ok { + // Imported wrappers may call the same target methods. Only + // accessors called by the formula package start matrix tracking. + switch matrixTargetCall(call) { + case optionsMatrix: + if fn.Pkg == pkg { + tracked = true + instrs = append(instrs, trackerCall(block, optionsTracker, call)) + } + case requireMatrix: + if fn.Pkg == pkg { + tracked = true + instrs = append(instrs, trackerCall(block, requireTracker, call)) + } + } + } + } + block.Instrs = instrs + } + } + return tracked +} + +func (t *tracker) matrix() classfile.Matrix { + t.mu.Lock() + defer t.mu.Unlock() + + t.active = false + t.maps = nil + return classfile.Matrix{ + Require: cloneKeys(t.require), + Options: cloneKeys(t.options), + } +} + +func (t *tracker) trackMap(kind matrixKind, values map[string][]string) { + if values == nil { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + if !t.active { + return + } + t.maps[reflect.ValueOf(values).Pointer()] = trackedMap{kind: kind, values: values} +} + +func (t *tracker) trackLookup(values map[string][]string, key string) { + if values == nil { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + if !t.active { + return + } + + tracked, ok := t.maps[reflect.ValueOf(values).Pointer()] + if !ok { + return + } + if tracked.kind == requireMatrix { + t.require[key] = struct{}{} + } else { + t.options[key] = struct{}{} + } +} + +func matrixTargetCall(call *ssa.Call) matrixKind { + callee := call.Call.StaticCallee() + if callee == nil || callee.Signature.Recv() == nil { + return unknownMatrix + } + + typ := types.Unalias(callee.Signature.Recv().Type()) + if ptr, ok := typ.(*types.Pointer); ok { + typ = types.Unalias(ptr.Elem()) + } + named, ok := typ.(*types.Named) + if !ok { + return unknownMatrix + } + obj := named.Obj() + if obj.Pkg() == nil || obj.Name() != "matrixTarget" || obj.Pkg().Path() != formulaPackagePath { + return unknownMatrix + } + + switch callee.Name() { + case "Options": + return optionsMatrix + case "Require": + return requireMatrix + default: + return unknownMatrix + } +} + +func isMatrixMap(typ types.Type) bool { + return types.Identical(types.Unalias(typ).Underlying(), matrixMapType) +} + +func trackerCall(block *ssa.BasicBlock, fn *ssa.Function, args ...ssa.Value) *ssa.Call { + call := &ssa.Call{Call: ssa.CallCommon{Value: fn, Args: args}} + // x/tools does not expose constructors for post-build instructions. + // Populate the metadata ixgo reads before appending the call to the block. + callValue := reflect.ValueOf(call).Elem() + register := callValue.FieldByName("register") + setUnexported(register.FieldByName("typ"), reflect.ValueOf(fn.Signature.Results())) + instruction := register.FieldByName("anInstruction") + setUnexported(instruction.FieldByName("block"), reflect.ValueOf(block)) + + for _, arg := range args { + if refs := arg.Referrers(); refs != nil { + *refs = append(*refs, call) + } + } + return call +} + +func setUnexported(dst, src reflect.Value) { + reflect.NewAt(dst.Type(), unsafe.Pointer(dst.UnsafeAddr())).Elem().Set(src) +} + +func cloneKeys(keys map[string]struct{}) map[string][]string { + if len(keys) == 0 { + return nil + } + matrix := make(map[string][]string, len(keys)) + for key := range keys { + matrix[key] = nil + } + return matrix +} diff --git a/internal/formula/tracker_test.go b/internal/formula/tracker_test.go new file mode 100644 index 00000000..65e57209 --- /dev/null +++ b/internal/formula/tracker_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package formula + +import ( + "io/fs" + "os" + "reflect" + "testing" +) + +func TestLoadFSMatrixTracker(t *testing.T) { + tests := []struct { + name string + path string + wantRequire map[string][]string + wantOptions map[string][]string + }{ + { + name: "data flow", + path: "trackcases_llar.gox", + wantRequire: map[string][]string{ + "direct-require": nil, + "helper-require": nil, + "same": nil, + }, + wantOptions: map[string][]string{ + "alias": nil, + "closure": nil, + "comma-ok": nil, + "direct-option": nil, + "dynamic": nil, + "helper-option": nil, + "interface": nil, + "named": nil, + "pointer": nil, + "returned": nil, + "same": nil, + "struct": nil, + "type-alias": nil, + }, + }, + { + name: "panic isolation", + path: "trackpanic_llar.gox", + wantOptions: map[string][]string{ + "after-panic": nil, + "before-panic": nil, + }, + }, + { + name: "filter is not probed", + path: "trackfilter_llar.gox", + }, + { + name: "no matrix access", + path: "hello_llar.gox", + }, + } + + fsys := os.DirFS("testdata/formula").(fs.ReadFileFS) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := LoadFS(fsys, tt.path) + if err != nil { + t.Fatalf("LoadFS(%q) failed: %v", tt.path, err) + } + if !reflect.DeepEqual(f.Matrix.Require, tt.wantRequire) { + t.Fatalf("Matrix.Require = %#v, want %#v", f.Matrix.Require, tt.wantRequire) + } + if !reflect.DeepEqual(f.Matrix.Options, tt.wantOptions) { + t.Fatalf("Matrix.Options = %#v, want %#v", f.Matrix.Options, tt.wantOptions) + } + }) + } +} From 14b1b1f45e29721f5da8801ceebc9de266d4da3b Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Sun, 12 Jul 2026 16:14:27 +0800 Subject: [PATCH 2/3] chore: keep tracker branch code-only --- AGENTS.md | 63 ----------------------------------- doc/matrix-options-tracker.md | 53 ----------------------------- 2 files changed, 116 deletions(-) delete mode 100644 doc/matrix-options-tracker.md diff --git a/AGENTS.md b/AGENTS.md index efd68cba..219b0c45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,69 +107,6 @@ func main() { 3. **Error Handling**: If a file doesn't match any registered classfile pattern (wrong suffix), `BuildFile` returns "undefined" errors for DSL functions -### Classfile `var` Fields - -In an XGo classfile, the first top-level `var (...)` declaration is the class -fields declaration. These names compile into fields on the generated struct, -not package-level variables. If the fields have initializers, XGo generates an -`XGo_Init()` method and calls it before the classfile body runs. - -Source file `hello_llar.gox`: -```gox -var ( - seen map[string]bool = {} -) - -func mark(key string) { - seen[key] = true -} - -id "DaveGamble/cJSON" -fromVer "v1.0.0" - -onRequire (proj, deps) => { - mark "zlib" -} -``` - -Generated Go code: -```go -package main - -import "github.com/goplus/llar/formula" - -type hello struct { - formula.ModuleF - seen map[string]bool -} - -func (this *hello) mark(key string) { - this.seen[key] = true -} - -func (this *hello) MainEntry() { - this.XGo_Init() - this.Id("DaveGamble/cJSON") - this.FromVer("v1.0.0") - this.OnRequire(func(proj *formula.Project, deps *formula.ModuleDeps) { - this.mark("zlib") - }) -} - -func (this *hello) Main() { - formula.Gopt_ModuleF_Main(this) -} - -func (this *hello) XGo_Init() *hello { - this.seen = map[string]bool{} - return this -} - -func main() { - new(hello).Main() -} -``` - ## Build & Test ```bash diff --git a/doc/matrix-options-tracker.md b/doc/matrix-options-tracker.md deleted file mode 100644 index 61e39f8b..00000000 --- a/doc/matrix-options-tracker.md +++ /dev/null @@ -1,53 +0,0 @@ -# Proposal: Discover Formula Matrix with SSA Instrumentation - -## Summary - -LLAR discovers matrix keys by observing formula execution. Formula code remains -unchanged; matrix reads are instrumented in Go SSA after ixgo has built the -package and before ixgo translates it for execution. - -## Pipeline - -```text -_llar.gox source - -> generated Go source - -> x/tools SSA and built-in optimization - -> tracker call insertion - -> ixgo interpreter -``` - -The tracker does not rewrite XGo AST or modify ixgo. - -## Instrumentation - -XGo lowers a matrix read into a method call followed by `ssa.Lookup`: - -```text -target.options[key] - -> matrixTarget.Options() - -> map lookup -``` - -The tracker inserts no-result calls at two points: - -1. After `matrixTarget.Options()` or `matrixTarget.Require()`, register the - returned map as options or require. -2. Before each `ssa.Lookup` on `map[string][]string`, record the key only when - the map was registered in step 1. - -Map identity follows normal assignments and function arguments, so aliases and -helpers do not require data-flow reconstruction. - -## Probe - -`LoadFS` executes the formula hooks with empty probe inputs, then copies the -observed require and options keys into `Formula.Matrix`. Values are filled later -from the selected matrix. Tracker calls are disabled after the probe and do not -affect normal builds. - -## Boundaries - -- Instrumentation runs after all x/tools SSA builder passes. -- Inserted calls do not produce values or change control flow. -- Only code represented in the ixgo SSA program can be instrumented. -- Additional probe rounds and candidate generation are separate concerns. From 11a8363ccd7cf28cc6c7b8292979a86b3bae2b4e Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Sun, 12 Jul 2026 16:52:19 +0800 Subject: [PATCH 3/3] fix(formula): isolate matrix probe SSA --- internal/formula/formula.go | 88 ++++++++++++++++++++++++++------ internal/formula/probe.go | 27 +++++++++- internal/formula/tracker_test.go | 48 +++++++++++++++++ 3 files changed, 147 insertions(+), 16 deletions(-) diff --git a/internal/formula/formula.go b/internal/formula/formula.go index 542d983d..b697dad2 100644 --- a/internal/formula/formula.go +++ b/internal/formula/formula.go @@ -10,11 +10,14 @@ import ( "io/fs" "path/filepath" "reflect" + "slices" "strings" "github.com/goplus/ixgo" "github.com/goplus/ixgo/xgobuild" "github.com/goplus/llar/formula" + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" _ "github.com/goplus/llar/internal/ixgo" ) @@ -35,6 +38,52 @@ type Formula struct { Filter func() bool } +type ssaState struct { + blocks map[*ssa.BasicBlock][]ssa.Instruction + referrers map[ssa.Value][]ssa.Instruction +} + +func saveSSAState(prog *ssa.Program) ssaState { + state := ssaState{ + blocks: make(map[*ssa.BasicBlock][]ssa.Instruction), + referrers: make(map[ssa.Value][]ssa.Instruction), + } + for fn := range ssautil.AllFunctions(prog) { + for _, block := range fn.Blocks { + state.blocks[block] = slices.Clone(block.Instrs) + for _, instr := range block.Instrs { + if value, ok := instr.(ssa.Value); ok { + state.saveReferrers(value) + } + for _, operand := range instr.Operands(nil) { + if operand != nil && *operand != nil { + state.saveReferrers(*operand) + } + } + } + } + } + return state +} + +func (s ssaState) saveReferrers(value ssa.Value) { + if _, ok := s.referrers[value]; ok { + return + } + if refs := value.Referrers(); refs != nil { + s.referrers[value] = slices.Clone(*refs) + } +} + +func (s ssaState) restore() { + for block, instrs := range s.blocks { + block.Instrs = instrs + } + for value, refs := range s.referrers { + *value.Referrers() = refs + } +} + // loadFS is the internal implementation for loading a formula from a filesystem. // It builds and interprets the formula file using the xgo classfile mechanism, // then extracts the struct fields containing module metadata and callbacks. @@ -95,10 +144,30 @@ func loadFS(fs fs.ReadFileFS, path string) (*Formula, error) { if err != nil { return nil, err } + state := saveSSAState(pkgs.Prog) tr := newTracker() tracked := tr.track(ctx, pkgs) - // Create a new interpreter for the loaded package + // Extract struct name from filename: "hello_llar.gox" -> "hello" + // The classfile mechanism generates a struct with this name + structName, _, ok := strings.Cut(filepath.Base(path), "_") + if !ok { + return nil, fmt.Errorf("failed to load formula: file name is not valid: %s", path) + } + + var matrix formula.Matrix + if tracked { + matrix, err = probeFormula(ctx, pkgs, structName, tr) + if err != nil { + return nil, err + } + } + + // NewInterp translates SSA eagerly. The probe interpreter retains the + // instrumented instructions, while the production interpreter below sees + // the original SSA restored from this snapshot. + state.restore() + interp, err := ctx.NewInterp(pkgs) if err != nil { return nil, err @@ -109,13 +178,6 @@ func loadFS(fs fs.ReadFileFS, path string) (*Formula, error) { return nil, err } - // Extract struct name from filename: "hello_llar.gox" -> "hello" - // The classfile mechanism generates a struct with this name - structName, _, ok := strings.Cut(filepath.Base(path), "_") - if !ok { - return nil, fmt.Errorf("failed to load formula: file name is not valid: %s", path) - } - // Get the generated struct type from the interpreter typ, ok := interp.GetType(structName) if !ok { @@ -135,20 +197,16 @@ func loadFS(fs fs.ReadFileFS, path string) (*Formula, error) { val.Interface().(interface{ Main() }).Main() // Extract the populated fields from the struct and return the Formula - loaded := &Formula{ + return &Formula{ structElem: class, ModPath: valueOf(class, "modPath").(string), FromVer: valueOf(class, "modFromVer").(string), + Matrix: matrix, OnBuild: valueOf(class, "fOnBuild").(func(*formula.Context, *formula.Project, *formula.BuildResult)), OnTest: valueOf(class, "fOnTest").(func(*formula.Context, *formula.Project, *formula.TestResult)), OnRequire: valueOf(class, "fOnRequire").(func(*formula.Project, *formula.ModuleDeps)), Filter: valueOf(class, "fFilter").(func() bool), - } - if tracked { - probeFormula(loaded) - } - loaded.Matrix = tr.matrix() - return loaded, nil + }, nil } // LoadFS loads a formula from a filesystem interface. diff --git a/internal/formula/probe.go b/internal/formula/probe.go index 3424f177..f7e0eea4 100644 --- a/internal/formula/probe.go +++ b/internal/formula/probe.go @@ -5,10 +5,14 @@ package formula import ( + "fmt" "io/fs" + "reflect" + "github.com/goplus/ixgo" classfile "github.com/goplus/llar/formula" "github.com/goplus/llar/mod/module" + "golang.org/x/tools/go/ssa" ) type probeFS struct{} @@ -21,7 +25,27 @@ func (probeFS) ReadFile(string) ([]byte, error) { return nil, fs.ErrNotExist } -func probeFormula(f *Formula) { +func probeFormula(ctx *ixgo.Context, pkg *ssa.Package, structName string, tr *tracker) (classfile.Matrix, error) { + interp, err := ctx.NewInterp(pkg) + if err != nil { + return classfile.Matrix{}, err + } + if err = interp.RunInit(); err != nil { + return classfile.Matrix{}, err + } + typ, ok := interp.GetType(structName) + if !ok { + return classfile.Matrix{}, fmt.Errorf("failed to load formula: struct name not found: %s", structName) + } + val := reflect.New(typ) + class := val.Elem() + val.Interface().(interface{ Main() }).Main() + f := &Formula{ + structElem: class, + OnBuild: valueOf(class, "fOnBuild").(func(*classfile.Context, *classfile.Project, *classfile.BuildResult)), + OnRequire: valueOf(class, "fOnRequire").(func(*classfile.Project, *classfile.ModuleDeps)), + } + originalTarget := valueOf(f.structElem, "target").(classfile.Matrix) // Probe with empty maps because discovery does not require a valid matrix. // A formula may read several independent options while configuring a build: @@ -57,6 +81,7 @@ func probeFormula(f *Formula) { f.OnBuild(ctx, project, &out) }) } + return tr.matrix(), nil } func safeProbeCall(call func()) { diff --git a/internal/formula/tracker_test.go b/internal/formula/tracker_test.go index 65e57209..4ce269e7 100644 --- a/internal/formula/tracker_test.go +++ b/internal/formula/tracker_test.go @@ -8,7 +8,11 @@ import ( "io/fs" "os" "reflect" + "slices" "testing" + + "github.com/goplus/ixgo" + "github.com/goplus/ixgo/xgobuild" ) func TestLoadFSMatrixTracker(t *testing.T) { @@ -76,3 +80,47 @@ func TestLoadFSMatrixTracker(t *testing.T) { }) } } + +func TestSSAStateRestore(t *testing.T) { + ctx := ixgo.NewContext(0) + content, err := os.ReadFile("testdata/formula/matrix_llar.gox") + if err != nil { + t.Fatal(err) + } + source, err := xgobuild.BuildFile(ctx, "matrix_llar.gox", content) + if err != nil { + t.Fatal(err) + } + pkg, err := ctx.LoadFile("main.go", source) + if err != nil { + t.Fatal(err) + } + + state := saveSSAState(pkg.Prog) + if !newTracker().track(ctx, pkg) { + t.Fatal("track returned false") + } + + changed := false + for block, instrs := range state.blocks { + if !slices.Equal(block.Instrs, instrs) { + changed = true + break + } + } + if !changed { + t.Fatal("tracker did not change SSA instructions") + } + + state.restore() + for block, instrs := range state.blocks { + if !slices.Equal(block.Instrs, instrs) { + t.Fatalf("block %d instructions were not restored", block.Index) + } + } + for value, refs := range state.referrers { + if !slices.Equal(*value.Referrers(), refs) { + t.Fatalf("referrers for %s were not restored", value.Name()) + } + } +}