diff --git a/cmd/llard/main.go b/cmd/llard/main.go index 564b8c8..b5ea23a 100644 --- a/cmd/llard/main.go +++ b/cmd/llard/main.go @@ -21,6 +21,7 @@ import ( "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/vcs" "github.com/joho/godotenv" + "github.com/prometheus/client_golang/prometheus/promhttp" ) type config struct { @@ -82,17 +83,23 @@ func run() error { WorkspaceDir: workspaceDir, Artifacts: artifacts, }) - handler := buildhttp.New(buildhttp.Options{ + buildHandler := buildhttp.New(buildhttp.Options{ FormulaStore: formulaStore, Cache: buildCache, Artifacts: artifacts, WorkspaceDir: workspaceDir, }) + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + mux.Handle("/v1/artifacts/", buildHandler) server := &http.Server{ Addr: cfg.addr, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { log.Printf("request method=%s uri=%s remote=%s", r.Method, r.URL.RequestURI(), r.RemoteAddr) - handler.ServeHTTP(w, r) + mux.ServeHTTP(w, r) }), } diff --git a/go.mod b/go.mod index 73da115..9238626 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/petermattis/goid v0.0.0-20260713073324-57ed88f34a0a + github.com/prometheus/client_golang v1.23.2 github.com/qiniu/go-sdk/v7 v7.26.14 github.com/qiniu/x v1.16.0 github.com/spf13/cobra v1.10.2 @@ -22,6 +23,8 @@ require ( require ( github.com/BurntSushi/toml v1.3.2 // indirect github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.18.1 // indirect github.com/docker/cli v29.0.3+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect @@ -35,8 +38,12 @@ require ( github.com/klauspost/compress v1.18.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/timandy/routine v1.1.5 // indirect @@ -45,7 +52,8 @@ require ( github.com/visualfc/gid v0.3.0 // indirect github.com/visualfc/goembed v0.3.2 // indirect github.com/visualfc/xtype v0.2.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/tools v0.40.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + google.golang.org/protobuf v1.36.8 // indirect modernc.org/fileutil v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index bbca9d9..b9b16de 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,10 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -48,15 +52,16 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -67,12 +72,20 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/qiniu/go-sdk/v7 v7.26.14 h1:kV9zdvTOM0w9/lgeYffDDw0Fj8FH14/XgCjk4Wb80k8= github.com/qiniu/go-sdk/v7 v7.26.14/go.mod h1:ri7fGwbio0pRDFr8EK5TUpx0DbnpIMJ2bMSDxGWfCbk= github.com/qiniu/x v1.16.0 h1:W2VOecyIT3Uxwjm6vJinUR7G3gpwgUgHZA9OpeHArdE= github.com/qiniu/x v1.16.0/go.mod h1:AiovSOCaRijaf3fj+0CBOpR1457pn24b0Vdb1JpwhII= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -88,8 +101,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/timandy/routine v1.1.5 h1:LSpm7Iijwb9imIPlucl4krpr2EeCeAUvifiQ9Uf5X+M= github.com/timandy/routine v1.1.5/go.mod h1:kXslgIosdY8LW0byTyPnenDgn4/azt2euufAq9rK51w= github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= @@ -102,6 +116,10 @@ github.com/visualfc/goembed v0.3.2 h1:a9m6o9VTzNk3mEF98C8cHp8f8P8BblyjsjajXGfTp8 github.com/visualfc/goembed v0.3.2/go.mod h1:jCVCz/yTJGyslo6Hta+pYxWWBuq9ADCcIVZBTQ0/iVI= github.com/visualfc/xtype v0.2.0 h1:0ESNXyWHtK01kaOzOyqHsR1ZjEPdNu/IWPZkf0VOHl8= github.com/visualfc/xtype v0.2.0/go.mod h1:183MDtzLqyDkCm5zCH42vJGq/aQE5W25k3Z6UOZxLF0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= @@ -112,6 +130,8 @@ golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/build/build.go b/internal/build/build.go index ee6d272..9e49593 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -10,11 +10,13 @@ import ( "path/filepath" "sort" "strings" + "time" classfile "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/build/cache" "github.com/goplus/llar/internal/execbroker" "github.com/goplus/llar/internal/formula/repo" + "github.com/goplus/llar/internal/metrics" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" @@ -239,14 +241,22 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul } deps := b.resolveModTransitiveDeps(targets, mod) modVer := module.Version{Path: mod.Path, Version: mod.Version} + cacheTarget := fmt.Sprintf("%s@%s?matrix=%s", mod.Path, mod.Version, b.matrix) // Consult the build cache. A hit means we already have the // module's build metadata and its installDir is populated from a // previous successful build. entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix}) if err != nil { + metrics.BuildCacheLookups.WithLabelValues(cacheTarget, "error").Inc() + metrics.BuildFailures.WithLabelValues("cache_get").Inc() return Result{}, err } + if cacheHit { + metrics.BuildCacheLookups.WithLabelValues(cacheTarget, "hit").Inc() + } else { + metrics.BuildCacheLookups.WithLabelValues(cacheTarget, "miss").Inc() + } // Fast path: cache hit and no OnTest to run. Skip source clone // and OnBuild entirely. @@ -269,9 +279,11 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul // TODO(MeteorsLiu): Support different code host repo, err := b.newRepo(fmt.Sprintf("github.com/%s", mod.Path)) if err != nil { + metrics.BuildFailures.WithLabelValues("source_sync").Inc() return Result{}, err } if err := repo.Sync(ctx, mod.Version, "", tmpSourceDir); err != nil { + metrics.BuildFailures.WithLabelValues("source_sync").Inc() return Result{}, err } @@ -305,6 +317,7 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul var out classfile.BuildResult mod.OnBuild(buildContext, project, &out) if len(out.Errs()) > 0 { + metrics.BuildFailures.WithLabelValues("on_build").Inc() return errors.Join(out.Errs()...) } metadata = out.Metadata() @@ -333,6 +346,7 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul Deps: deps, }) if err != nil { + metrics.BuildFailures.WithLabelValues("cache_put").Inc() return Result{}, err } metadata = entry.Metadata @@ -369,8 +383,13 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul sort.Strings(lockPaths) unlocks := make([]func(), 0, len(lockPaths)) for _, path := range lockPaths { + started := time.Now() + metrics.BuildLockWaiters.WithLabelValues(path).Inc() unlock, err := b.store.LockModule(path) + metrics.BuildLockWaiters.WithLabelValues(path).Dec() + metrics.BuildLockWaitDuration.Observe(time.Since(started).Seconds()) if err != nil { + metrics.BuildFailures.WithLabelValues("lock").Inc() for i := len(unlocks) - 1; i >= 0; i-- { unlocks[i]() } diff --git a/internal/build/http/http.go b/internal/build/http/http.go index 39740e7..34ccf4b 100644 --- a/internal/build/http/http.go +++ b/internal/build/http/http.go @@ -8,18 +8,21 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" "strings" "sync" + "time" "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/artifact" "github.com/goplus/llar/internal/build" "github.com/goplus/llar/internal/build/cache" "github.com/goplus/llar/internal/formula/repo" + "github.com/goplus/llar/internal/metrics" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/mod/module" "golang.org/x/sync/singleflight" @@ -72,8 +75,18 @@ func New(opts Options) http.Handler { } func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + started := time.Now() + outcome := "success" + metrics.RequestsInFlight.Inc() + defer func() { + metrics.RequestsInFlight.Dec() + metrics.RequestsTotal.WithLabelValues(outcome).Inc() + metrics.RequestDuration.WithLabelValues(outcome).Observe(time.Since(started).Seconds()) + }() + w.Header().Set("Content-Type", "application/x-cmdjsonl") if r.Method != http.MethodGet { + outcome = "client_error" w.Header().Set("Allow", http.MethodGet) writeCommand(w, "error", "method not allowed") return @@ -81,6 +94,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { req, err := parseRequest(r) if err != nil { + outcome = "client_error" writeCommand(w, "error", err.Error()) return } @@ -89,6 +103,10 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { result, err := h.do(r.Context(), req, info) info.flush() if err != nil { + outcome = "server_error" + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + outcome = "canceled" + } writeCommand(w, "error", err.Error()) return } @@ -99,14 +117,29 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *handler) do(ctx context.Context, req request, info io.Writer) (result, error) { key := requestKey(req) - stream, _ := h.infos.LoadOrStore(key, newFanout()) + stream, shared := h.infos.LoadOrStore(key, newFanout()) + if shared { + metrics.BuildSharedRequests.Inc() + } fanout := stream.(*fanout) remove := fanout.add(info) defer remove() call := h.group.DoChan(key, func() (any, error) { + started := time.Now() + outcome := "success" + metrics.BuildsInProgress.WithLabelValues(key).Inc() + defer func() { + metrics.BuildsInProgress.WithLabelValues(key).Dec() + metrics.BuildsTotal.WithLabelValues(outcome).Inc() + metrics.BuildDuration.WithLabelValues(outcome).Observe(time.Since(started).Seconds()) + }() defer h.infos.Delete(key) - return h.build(context.WithoutCancel(ctx), req, fanout) + result, err := h.build(context.WithoutCancel(ctx), req, fanout) + if err != nil { + outcome = "server_error" + } + return result, err }) select { case <-ctx.Done(): @@ -125,6 +158,7 @@ func (h *handler) build(ctx context.Context, req request, info io.Writer) (resul Matrix: req.matrix, }) if err != nil { + metrics.BuildFailures.WithLabelValues("resolve").Inc() return result{}, err } @@ -153,6 +187,7 @@ func (h *handler) build(ctx context.Context, req request, info io.Writer) (resul MatrixStr: req.matrixStr, }) if err != nil { + metrics.BuildFailures.WithLabelValues("artifact_get").Inc() return result{}, err } item := artifactMessage{ diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..5928f3f --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,74 @@ +// Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + requestDurationBuckets = []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1800, 3600} + lockWaitBuckets = []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300} + + RequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "llard", + Name: "requests_total", + Help: "Total number of artifact requests by outcome.", + }, []string{"outcome"}) + RequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "llard", + Name: "request_duration_seconds", + Help: "Artifact request duration in seconds by outcome.", + Buckets: requestDurationBuckets, + }, []string{"outcome"}) + RequestsInFlight = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "llard", + Name: "requests_in_flight", + Help: "Current number of artifact requests being served.", + }) + BuildsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "llard", + Name: "builds_total", + Help: "Total number of actual builds after request coalescing by outcome.", + }, []string{"outcome"}) + BuildDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "llard", + Name: "build_duration_seconds", + Help: "Actual build duration in seconds by outcome.", + Buckets: requestDurationBuckets, + }, []string{"outcome"}) + BuildsInProgress = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "llard", + Name: "builds_in_progress", + Help: "Current number of actual builds by target.", + }, []string{"target"}) + BuildSharedRequests = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "llard", + Name: "build_shared_requests_total", + Help: "Total number of requests coalesced into an existing build.", + }) + BuildCacheLookups = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "llard", + Name: "build_cache_lookups_total", + Help: "Total number of build cache lookups by target and result.", + }, []string{"target", "result"}) + BuildLockWaiters = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "llard", + Name: "build_lock_waiters", + Help: "Current number of builds waiting to acquire a module lock by target.", + }, []string{"target"}) + BuildLockWaitDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "llard", + Name: "build_lock_wait_duration_seconds", + Help: "Build lock acquisition duration in seconds.", + Buckets: lockWaitBuckets, + }) + BuildFailures = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "llard", + Name: "build_failures_total", + Help: "Total number of build pipeline failures by stage.", + }, []string{"stage"}) +) diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..68952a0 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestCollectorsRegistered(t *testing.T) { + RequestsTotal.WithLabelValues("test") + RequestDuration.WithLabelValues("test") + BuildsTotal.WithLabelValues("test") + BuildDuration.WithLabelValues("test") + BuildsInProgress.WithLabelValues("test") + BuildCacheLookups.WithLabelValues("test", "hit") + BuildLockWaiters.WithLabelValues("test") + BuildFailures.WithLabelValues("test") + + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + registered := make(map[string]bool, len(families)) + for _, family := range families { + registered[family.GetName()] = true + } + for _, name := range []string{ + "llard_requests_total", + "llard_request_duration_seconds", + "llard_requests_in_flight", + "llard_builds_total", + "llard_build_duration_seconds", + "llard_builds_in_progress", + "llard_build_shared_requests_total", + "llard_build_cache_lookups_total", + "llard_build_lock_waiters", + "llard_build_lock_wait_duration_seconds", + "llard_build_failures_total", + } { + if !registered[name] { + t.Errorf("metric %q is not registered", name) + } + } +}