diff --git a/.dockerignore b/.dockerignore index 91f5fa94ae..831f07e852 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,7 @@ web/node_modules web/dist web/assets-source silo +silo-darwin-arm64 streamapp *.md .playwright-cli/ diff --git a/.github/actions/ci-tools/go-checks.mk b/.github/actions/ci-tools/go-checks.mk index 7ad94a6485..2bdf1b8e6b 100644 --- a/.github/actions/ci-tools/go-checks.mk +++ b/.github/actions/ci-tools/go-checks.mk @@ -12,9 +12,13 @@ lint: $(MAKE) -f Makefile lint-router-recovery # The whole tree, so a change that breaks vet in a package it did not touch -# still fails; lint-changed reports only changed lines. +# still fails; lint-changed reports only changed lines. The macOS node build +# is vetted too, since Linux never compiles the darwin installer, and its +# cross-build fails on a new cgo import or a gap in imageutil's node stub. vet: go vet ./... + GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go vet -tags silo_node ./cmd/silo/ ./internal/nodeinstall/ ./internal/nodebuild/ ./internal/imageutil/ ./web/ + $(MAKE) -f Makefile build-node-darwin NODE_DARWIN_OUT=/dev/null # Package assertions belong to the tests lane; these check generated artifacts. contracts: diff --git a/.gitignore b/.gitignore index dd5678dd07..92e1cfc2bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Binary /silo +/silo-darwin-arm64 *.exe # IDE diff --git a/Dockerfile b/Dockerfile index 453f0940ba..476a72b216 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,32 @@ RUN --network=none go build \ -ldflags "-X github.com/Silo-Server/silo-server/internal/buildinfo.revisionOverride=${BUILD_REVISION} -X github.com/Silo-Server/silo-server/internal/buildinfo.dirtyOverride=${BUILD_DIRTY} -X github.com/Silo-Server/silo-server/internal/buildinfo.buildNumberOverride=${BUILD_NUMBER} -X github.com/Silo-Server/silo-server/internal/buildinfo.builtAtOverride=${BUILD_DATE}" \ -o /silo ./cmd/silo/ +# The macOS transcode node (darwin/arm64), cross-compiled on the build host. +# The silo_node tag leaves out libvips and the web app, so it needs no cgo and +# links only system frameworks; the Go linker signs it ad hoc. A Mac copies it +# out of the image so it matches this server (docs/architecture/transcode-nodes.md). +FROM --platform=$BUILDPLATFORM golang:1.27 AS node_darwin +ENV GOPROXY=https://proxy.golang.org,direct +ENV GOPRIVATE=github.com/Silo-Server/* +ENV GONOSUMDB=github.com/Silo-Server/* +WORKDIR /app +COPY go.mod go.sum ./ +COPY internal/compat/zishang520-webtransport-go/ internal/compat/zishang520-webtransport-go/ +RUN go mod download +COPY internal/ internal/ +COPY migrations/ migrations/ +COPY contracts/ contracts/ +COPY cmd/ cmd/ +COPY web/embed.go web/embed_node.go web/ +RUN --network=none CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -tags silo_node -trimpath -o /dev/null ./cmd/silo/ +ARG BUILD_REVISION +ARG BUILD_DIRTY=false +ARG BUILD_NUMBER +ARG BUILD_DATE +RUN --network=none CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -tags silo_node -trimpath \ + -ldflags "-s -w -X github.com/Silo-Server/silo-server/internal/buildinfo.revisionOverride=${BUILD_REVISION} -X github.com/Silo-Server/silo-server/internal/buildinfo.dirtyOverride=${BUILD_DIRTY} -X github.com/Silo-Server/silo-server/internal/buildinfo.buildNumberOverride=${BUILD_NUMBER} -X github.com/Silo-Server/silo-server/internal/buildinfo.builtAtOverride=${BUILD_DATE}" \ + -o /silo-darwin-arm64 ./cmd/silo/ + # Stage 3: Runtime # CI rebuilds this stage without the layer cache (no-cache-filters: runtime in # trusted-linux-build.yml) so every image picks up Debian security updates; the @@ -134,6 +160,8 @@ RUN ln -sf ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ ENV NODE_OPTIONS=--use-openssl-ca LABEL io.silo.intel-qsv-runtime="${INTEL_QSV_RUNTIME}" COPY --from=build /silo /usr/local/bin/silo +# Not run here: a Mac copies it out to run a transcode node with VideoToolbox. +COPY --from=node_darwin /silo-darwin-arm64 /usr/share/silo/nodes/silo-darwin-arm64 EXPOSE 8080 8096 13378 # Silo runs database migrations before its HTTP listener starts, and an # upgrade on a large library can migrate for many minutes. The start period diff --git a/Makefile b/Makefile index 448eebb210..cb3f6e772e 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,14 @@ frontend: build: frontend go build -ldflags "$(GO_LDFLAGS)" -o silo ./cmd/silo/ +# Cross-build the macOS transcode node as the image's node_darwin stage does. +# Without cgo, a new cgo import or an imageutil function missing from its node +# stub fails here before the image build does. +.PHONY: build-node-darwin +NODE_DARWIN_OUT ?= silo-darwin-arm64 +build-node-darwin: embed-stub + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -tags silo_node -trimpath -ldflags "-s -w $(GO_LDFLAGS)" -o "$(NODE_DARWIN_OUT)" ./cmd/silo/ + # Run frontend dev server (proxies API to localhost:8080) dev-frontend: cd web && pnpm run dev diff --git a/cmd/silo/main.go b/cmd/silo/main.go index fd942a712a..94fd43a372 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -104,7 +104,9 @@ import ( _ "github.com/Silo-Server/silo-server/internal/metadata/nfo" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/nodeauth" + "github.com/Silo-Server/silo-server/internal/nodebuild" "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodeinstall" "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/noderecipe" @@ -158,7 +160,9 @@ import ( siloweb "github.com/Silo-Server/silo-server/web" ) -const serverModeTranscode = "transcode" +// nodeBuildOnly is what a node build answers anything but a transcode node +// run or a "silo node" subcommand. +const nodeBuildOnly = "this silo build runs only as a transcode node: install it with `silo node install`, or set MODE=transcode" // serverModeProxy is MODE for a proxy node. const serverModeProxy = "proxy" @@ -801,6 +805,11 @@ func addRedisBootstrapSettings(redisURL, redisURLVar string, configured map[stri // main starts the Silo server or a requested maintenance command. func main() { + // A node build has no image processing or web app, so it runs no + // subcommand but its own; flags still reach the transcode node. + if nodebuild.Node && len(os.Args) > 1 && os.Args[1] != "node" && !strings.HasPrefix(os.Args[1], "-") { + log.Fatal(nodeBuildOnly) + } var storageAdmission *pglock.NodeAdmission // The admission session is detached from the pool and lives until process // exit. Worker Stop methods do not all wait for in-flight writes, so closing @@ -872,6 +881,14 @@ func main() { return } + if len(os.Args) > 1 && os.Args[1] == "node" { + if err := nodeinstall.Run(os.Args[2:], os.Stdin, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "silo node: %v\n", err) + os.Exit(1) + } + return + } + envFile := flag.String("env", ".env", "path to .env bootstrap file") migrateOnly := flag.Bool("migrate-only", false, "apply database migrations and exit") migrateStatus := flag.Bool("migrate-status", false, "show database migration status and exit") @@ -905,6 +922,9 @@ func main() { runTranscodeNode() return } + if nodebuild.Node { + log.Fatal(nodeBuildOnly) + } // Step 1: Bootstrap from .env bc, err := config.LoadBootstrap(*envFile) diff --git a/cmd/silo/node_drain.go b/cmd/silo/node_drain.go index b14c597f01..8ace32447a 100644 --- a/cmd/silo/node_drain.go +++ b/cmd/silo/node_drain.go @@ -3,7 +3,6 @@ package main import ( "context" "log/slog" - "math" "os" "strconv" "strings" @@ -15,14 +14,6 @@ import ( "github.com/Silo-Server/silo-server/internal/transcodenode" ) -// nodeDrainTimeoutEnv bounds how long a transcode node that received SIGTERM -// keeps serving its sessions before it shuts down. 0 turns the drain off. -const nodeDrainTimeoutEnv = "SILO_NODE_DRAIN_TIMEOUT" - -// defaultNodeDrainTimeout lets most episodes and a long tail of movie sessions -// finish, while keeping a rolling upgrade of several nodes bounded. -const defaultNodeDrainTimeout = 10 * time.Minute - // nodeDrainKeyCheckInterval is how often a SIGTERM drain asks whether the // main server still accepts this node's key. The answer is an atomic read. const nodeDrainKeyCheckInterval = time.Second @@ -44,28 +35,22 @@ type nodeDrain struct { // timeout, or not at all when it is 0. keyRejected ends the wait early; see // nodeDrain.keyRejected. func newTranscodeNodeDrain(srv *transcodenode.Server, keyRejected func() bool) *nodeDrain { - timeout := nodeDrainTimeout(os.Getenv(nodeDrainTimeoutEnv)) + timeout := nodeDrainTimeout(os.Getenv(transcodenode.EnvDrainTimeout)) if timeout <= 0 { return nil } return &nodeDrain{timeout: timeout, begin: srv.BeginShutdownDrain, wait: srv.WaitDrained, activeJobs: srv.ActiveJobs, keyRejected: keyRejected} } -// nodeDrainTimeout parses SILO_NODE_DRAIN_TIMEOUT: a Go duration ("15m") or -// whole seconds ("900"). Empty or invalid means the default. +// nodeDrainTimeout parses SILO_NODE_DRAIN_TIMEOUT +// (transcodenode.ParseDrainTimeout). Invalid means the default. func nodeDrainTimeout(value string) time.Duration { - value = strings.TrimSpace(value) - if value == "" { - return defaultNodeDrainTimeout - } - if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 && seconds <= math.MaxInt64/int(time.Second) { - return time.Duration(seconds) * time.Second - } - if timeout, err := time.ParseDuration(value); err == nil && timeout >= 0 { - return timeout + timeout, err := transcodenode.ParseDrainTimeout(value) + if err != nil { + slog.Warn("invalid node drain timeout; using the default", "env", transcodenode.EnvDrainTimeout, "value", strings.TrimSpace(value), "default", transcodenode.DefaultDrainTimeout) + return transcodenode.DefaultDrainTimeout } - slog.Warn("invalid node drain timeout; using the default", "env", nodeDrainTimeoutEnv, "value", value, "default", defaultNodeDrainTimeout) - return defaultNodeDrainTimeout + return timeout } // run drains the node: it stops taking new work and waits until its work is diff --git a/cmd/silo/node_drain_test.go b/cmd/silo/node_drain_test.go index f5bb883de0..64256a1210 100644 --- a/cmd/silo/node_drain_test.go +++ b/cmd/silo/node_drain_test.go @@ -8,6 +8,8 @@ import ( "testing" "testing/synctest" "time" + + "github.com/Silo-Server/silo-server/internal/transcodenode" ) func TestNodeDrainTimeout(t *testing.T) { @@ -15,15 +17,15 @@ func TestNodeDrainTimeout(t *testing.T) { value string want time.Duration }{ - {"", defaultNodeDrainTimeout}, + {"", transcodenode.DefaultDrainTimeout}, {"0", 0}, {"0s", 0}, {"900", 15 * time.Minute}, {"45m", 45 * time.Minute}, {" 2h ", 2 * time.Hour}, - {"-5m", defaultNodeDrainTimeout}, - {"99999999999", defaultNodeDrainTimeout}, // more seconds than a Duration holds - {"soon", defaultNodeDrainTimeout}, + {"-5m", transcodenode.DefaultDrainTimeout}, + {"99999999999", transcodenode.DefaultDrainTimeout}, // more seconds than a Duration holds + {"soon", transcodenode.DefaultDrainTimeout}, } { if got := nodeDrainTimeout(test.value); got != test.want { t.Errorf("nodeDrainTimeout(%q) = %v, want %v", test.value, got, test.want) diff --git a/cmd/silo/node_keys.go b/cmd/silo/node_keys.go index e173a97095..8942c965b2 100644 --- a/cmd/silo/node_keys.go +++ b/cmd/silo/node_keys.go @@ -12,6 +12,7 @@ import ( "github.com/Silo-Server/silo-server/internal/envutil" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/secret" + "github.com/Silo-Server/silo-server/internal/transcodenode" ) // startProxyNodeKeyRegistry loads a proxy's copy of the stream node credential @@ -38,16 +39,12 @@ func startProxyNodeKeyRegistry(ctx context.Context, pool *pgxpool.Pool, cipher * return registry } -// envNodeKey is the transcode node's own key, issued by an administrator. The -// node verifies every request the core signs to it with this key. -const envNodeKey = "SILO_NODE_KEY" - // transcodeNodeKey reads and validates SILO_NODE_KEY, or the file // SILO_NODE_KEY_FILE names. A transcode node refuses to start without one, or // with one that is not a well-formed node key: it could verify nothing, so // every request to it would fail. func transcodeNodeKey(getenv func(string) string) (string, error) { - key, usedVar, err := envutil.LookupSecret(getenv, envNodeKey) + key, usedVar, err := envutil.LookupSecret(getenv, transcodenode.EnvNodeKey) if err != nil { return "", err } diff --git a/cmd/silo/transcode_node.go b/cmd/silo/transcode_node.go index f96377600f..ed8bb897d1 100644 --- a/cmd/silo/transcode_node.go +++ b/cmd/silo/transcode_node.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "net" - "net/url" "os" "os/signal" "strings" @@ -35,11 +34,6 @@ import ( "github.com/Silo-Server/silo-server/internal/tuning" ) -// envCoreURL is the URL a transcode node reaches the core at. With its own -// key (envNodeKey) it is all the node needs: everything else comes from the -// core. -const envCoreURL = "SILO_CORE_URL" - // transcodeNodeIgnoredEnv are variables a transcode node once read and now // takes from the core or no longer needs: the database and Redis it read // settings, recipes and deny markers from, the data cipher's key, and the URL @@ -77,7 +71,7 @@ func transcodeNodeMode(envFile string) bool { if envFile != "" { _ = godotenv.Load(envFile) } - return strings.TrimSpace(os.Getenv("MODE")) == serverModeTranscode + return strings.TrimSpace(os.Getenv("MODE")) == transcodenode.Mode } // transcodeNodeEnv is what a transcode node reads from its environment. @@ -98,13 +92,13 @@ type transcodeNodeEnv struct { // URL or key, or a key that is not a well-formed node key, is an error: the // node could not even ask the core for its configuration. func loadTranscodeNodeEnv(getenv func(string) string) (transcodeNodeEnv, error) { - env := transcodeNodeEnv{coreURL: strings.TrimSpace(getenv(envCoreURL))} + env := transcodeNodeEnv{coreURL: strings.TrimSpace(getenv(transcodenode.EnvCoreURL))} if env.coreURL == "" { - return transcodeNodeEnv{}, fmt.Errorf("%s is required: the URL this node reaches the main server at", envCoreURL) + return transcodeNodeEnv{}, fmt.Errorf("%s is required: the URL this node reaches the main server at", transcodenode.EnvCoreURL) } - parsed, err := url.Parse(env.coreURL) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { - return transcodeNodeEnv{}, fmt.Errorf("%s must be an absolute http or https URL", envCoreURL) + parsed, ok := transcodenode.ParseCoreURL(env.coreURL) + if !ok { + return transcodeNodeEnv{}, fmt.Errorf("%s must be an absolute http or https URL", transcodenode.EnvCoreURL) } env.insecureCoreURL = parsed.Scheme == "http" && !loopbackHost(parsed.Hostname()) key, err := transcodeNodeKey(getenv) @@ -200,7 +194,7 @@ func runTranscodeNode() { stopLogging := configureTranscodeNodeLogging(appCtx, cfg, remote, client, nodeName) defer stopLogging() - slog.Info("silo starting", "mode", serverModeTranscode, "listen", cfg.Server.Listen, "log_level", cfg.Server.LogLevel, + slog.Info("silo starting", "mode", transcodenode.Mode, "listen", cfg.Server.Listen, "log_level", cfg.Server.LogLevel, "node_id", client.NodeID(), "node_name", nodeName) // Every job's input opens through the core with the grant the job @@ -269,7 +263,7 @@ func runTranscodeNode() { Congestion: tuningCfg.TCPCongestion == tuning.TCPCongestionBBR, SendBuffer: tuningCfg.TCPSendBuffer, } - startStandaloneServer(serverModeTranscode, cfg.Server.Listen, srv.Handler(), appCancel, shutdown, + startStandaloneServer(transcodenode.Mode, cfg.Server.Listen, srv.Handler(), appCancel, shutdown, standaloneServerHooks{drain: newTranscodeNodeDrain(srv, remote.KeyRejected)}, opts) } diff --git a/cmd/silo/transcode_node_test.go b/cmd/silo/transcode_node_test.go index 716963e6f2..4a29e3885a 100644 --- a/cmd/silo/transcode_node_test.go +++ b/cmd/silo/transcode_node_test.go @@ -46,7 +46,7 @@ func TestLoadTranscodeNodeEnvReadsKeyFile(t *testing.T) { if err := os.WriteFile(path, []byte(key+"\n"), 0o600); err != nil { t.Fatal(err) } - env := map[string]string{envCoreURL: "http://core:8080", envNodeKey + "_FILE": path} + env := map[string]string{transcodenode.EnvCoreURL: "http://core:8080", transcodenode.EnvNodeKey + "_FILE": path} got, err := loadTranscodeNodeEnv(func(name string) string { return env[name] }) if err != nil || got.nodeKey != key { t.Fatalf("node key = %q, %v; want the file's key", got.nodeKey, err) @@ -59,18 +59,18 @@ func TestLoadTranscodeNodeEnv(t *testing.T) { return func(name string) string { return values[name] } } for name, values := range map[string]map[string]string{ - "no core URL": {envNodeKey: key}, - "relative core URL": {envCoreURL: "core:8080", envNodeKey: key}, - "no key": {envCoreURL: "http://core:8080"}, - "malformed key": {envCoreURL: "http://core:8080", envNodeKey: "snk1_6_not-a-key"}, - "not a node key": {envCoreURL: "http://core:8080", envNodeKey: "a-shared-secret"}, + "no core URL": {transcodenode.EnvNodeKey: key}, + "relative core URL": {transcodenode.EnvCoreURL: "core:8080", transcodenode.EnvNodeKey: key}, + "no key": {transcodenode.EnvCoreURL: "http://core:8080"}, + "malformed key": {transcodenode.EnvCoreURL: "http://core:8080", transcodenode.EnvNodeKey: "snk1_6_not-a-key"}, + "not a node key": {transcodenode.EnvCoreURL: "http://core:8080", transcodenode.EnvNodeKey: "a-shared-secret"}, } { if _, err := loadTranscodeNodeEnv(env(values)); err == nil { t.Fatalf("%s: accepted", name) } } got, err := loadTranscodeNodeEnv(env(map[string]string{ - envCoreURL: " http://core:8080 ", envNodeKey: key + "\n", "PORT": "9100", + transcodenode.EnvCoreURL: " http://core:8080 ", transcodenode.EnvNodeKey: key + "\n", "PORT": "9100", envSecretKey: "shared-env-file", envNodeName: "gpu-6", envDatabaseURL: "postgres://interim", envRedisURL: "redis://shared", })) if err != nil { @@ -86,7 +86,7 @@ func TestLoadTranscodeNodeEnv(t *testing.T) { "https://core.example": false, "http://127.0.0.1:8080": false, "http://localhost:8080": false, "http://[::1]:8080": false, "http://192.168.1.10:8080": true, } { - loaded, err := loadTranscodeNodeEnv(env(map[string]string{envCoreURL: coreURL, envNodeKey: key})) + loaded, err := loadTranscodeNodeEnv(env(map[string]string{transcodenode.EnvCoreURL: coreURL, transcodenode.EnvNodeKey: key})) if err != nil || loaded.insecureCoreURL != insecure || loaded.listen != ":8080" { t.Fatalf("%s: insecure = %v, listen %q (%v); want insecure %v on :8080", coreURL, loaded.insecureCoreURL, loaded.listen, err, insecure) } @@ -161,22 +161,22 @@ func TestTranscodeNodeStartsFromTheCoreAlone(t *testing.T) { var env []string for _, kv := range os.Environ() { name, _, _ := strings.Cut(kv, "=") - if !slices.Contains(append(slices.Clone(transcodeNodeIgnoredEnv), envCoreURL, envNodeKey, envNodeKey+"_FILE", "PORT"), name) { + if !slices.Contains(append(slices.Clone(transcodeNodeIgnoredEnv), transcodenode.EnvCoreURL, transcodenode.EnvNodeKey, transcodenode.EnvNodeKey+"_FILE", "PORT"), name) { env = append(env, kv) } } - return append(append(env, "SILO_TEST_TRANSCODE_NODE=1", "MODE=transcode", nodeDrainTimeoutEnv+"=0"), extra...) + return append(append(env, "SILO_TEST_TRANSCODE_NODE=1", "MODE=transcode", transcodenode.EnvDrainTimeout+"=0"), extra...) } for name, key := range map[string]string{"missing key": "", "malformed key": "snk1_6_not-a-key"} { cmd := exec.Command(os.Args[0], "-test.run=^TestTranscodeNodeStartsFromTheCoreAlone$") - cmd.Env = nodeEnv(envCoreURL+"=http://127.0.0.1:1", envNodeKey+"="+key) + cmd.Env = nodeEnv(transcodenode.EnvCoreURL+"=http://127.0.0.1:1", transcodenode.EnvNodeKey+"="+key) output, err := cmd.CombinedOutput() if exit, ok := errors.AsType[*exec.ExitError](err); !ok || exit.ExitCode() == 0 { t.Fatalf("%s: process ended with %v, want a failed exit\n%s", name, err, output) } - if !strings.Contains(string(output), envNodeKey) { - t.Fatalf("%s: output does not name %s:\n%s", name, envNodeKey, output) + if !strings.Contains(string(output), transcodenode.EnvNodeKey) { + t.Fatalf("%s: output does not name %s:\n%s", name, transcodenode.EnvNodeKey, output) } } @@ -184,7 +184,7 @@ func TestTranscodeNodeStartsFromTheCoreAlone(t *testing.T) { core := newFakeNodeCore(t, key) port := freePort(t) cmd := exec.Command(os.Args[0], "-test.run=^TestTranscodeNodeStartsFromTheCoreAlone$") - cmd.Env = nodeEnv(envCoreURL+"="+core.URL, envNodeKey+"="+key, "PORT="+port) + cmd.Env = nodeEnv(transcodenode.EnvCoreURL+"="+core.URL, transcodenode.EnvNodeKey+"="+key, "PORT="+port) var output syncBuffer cmd.Stdout, cmd.Stderr = &output, &output if err := cmd.Start(); err != nil { diff --git a/docs/architecture/transcode-nodes.md b/docs/architecture/transcode-nodes.md index 97f2ed8d62..eab0133d86 100644 --- a/docs/architecture/transcode-nodes.md +++ b/docs/architecture/transcode-nodes.md @@ -278,7 +278,7 @@ cipher or event bus. Before its first configuration read the node sends its declaration ([Registration](#registration)), so the configuration it loads already reflects it. The node then reads its configuration, retrying with backoff from one second to 30 -seconds until a read succeeds or it is signalled, and only then opens its +seconds until a read succeeds or it is signaled, and only then opens its listener. A configuration that names another node id than the key is refused (`Client.FetchConfig`). Each call to the core has a 10-second timeout (`nodecontrol.Client`). @@ -514,6 +514,112 @@ node and a clock more than 30 seconds off as the causes; it still records the no drain state. A binary from before this design has no version, so it is never placed. +## macOS node build + +Docker on a Mac runs Linux in a VM with no VideoToolbox, so a Mac transcodes +with a native node beside a server in Docker. The base image (and the omnibus +image built on it) carries that node at +`/usr/share/silo/nodes/silo-darwin-arm64`, from the `node_darwin` stage of the +root `Dockerfile`: `CGO_ENABLED=0 GOOS=darwin GOARCH=arm64` with +`-tags silo_node`, cross-compiled on the build host and signed ad hoc by the Go +linker. The binary is the image's own revision, so a node copied from an image +matches the server that runs it. `make build-node-darwin` runs the same build; +the Go vet CI job runs it and vets the node build for darwin, which no Linux +build compiles. + +- **The `silo_node` tag** leaves libvips and the web app out. + `internal/imageutil` keeps the declarations both builds share in + `shared.go`, and the node build replaces the functions with + `imageutil_node.go`, the same exported API with every function failing. + `web/embed_node.go` replaces `web/embed.go` with an empty `DistFS`. A node + processes no images: its one image job, `chapterthumbs.ExtractFrame`, is + FFmpeg's. The binary links only system frameworks, so no Homebrew library + can stop it from loading. A new exported `imageutil` function needs a stub, + and a new cgo import the node links fails the build. +- **Node builds run only as a node.** `main` refuses every subcommand but + `silo node`, and, after the transcode branch, any run without + `MODE=transcode` (`nodebuild.Node`, `internal/nodebuild/node.go`). +- **Version.** `silo node version` and the installer name the build as Admin + shows the server's and each node's: the build number and the display + revision. +- **Installer.** `silo node install` and `silo node uninstall` + (`internal/nodeinstall/node_command.go`; the install, swap and file work + in `node_install_darwin.go`, launchctl in `launchd_darwin.go`) manage one + per-user LaunchAgent, + `org.siloserver.node`, and nothing system-wide. The cgo darwin build refuses + to install itself, since it loads Homebrew libraries, and other platforms + refuse both commands. + +| File | | +|---|---| +| `~/Library/Application Support/Silo/node/silo` | The binary. | +| `~/Library/Application Support/Silo/node/node.env` | Mode 0600. The installer manages `MODE=transcode`, `SILO_CORE_URL`, `SILO_NODE_KEY_FILE` naming `node-key`, and `PORT` (default 8100, since Docker Desktop often publishes 8080). Any other variable in the file, such as one from [Startup and environment](#startup-and-environment), is kept across reruns, except `SILO_NODE_KEY`, which would clash with the key file. | +| `~/Library/Application Support/Silo/node/node-key` | Mode 0600: the node key. | +| `~/Library/LaunchAgents/org.siloserver.node.plist` | Runs the binary with `-env` and `node.env`, `RunAtLoad`, `KeepAlive` with `SuccessfulExit` false (launchd restarts the node after a crash or a non-zero exit, not after the clean exit that ends a drain), and an `ExitTimeOut` of the drain timeout `node.env` sets plus a minute (660 seconds with the default). Current macOS caps `ExitTimeOut` at 60 seconds, so a logout or shutdown stops the node's streams a minute after SIGTERM; the installer's own stop does not depend on it. | +| `~/Library/Application Support/Silo/node-install.lock` | Mode 0600, in a 0700 folder: the lock every install and uninstall holds. It sits outside the node's folder, which uninstall removes while it holds the lock, and is never removed, so two runs always lock the same file. | +| `~/Library/Logs/Silo/node.log` | The node's output. Uninstall keeps it. | + +An install is also the upgrade: a rerun takes every value its flags leave out +from the existing install, and replaces the binary unless it runs from the +installed one. The key is never a command-line argument, which other users of +the Mac could read: a first install, and a rerun with `--new-key`, ask for it at +a hidden prompt, or read it from standard input when that isn't a terminal. It checks everything it can before it writes anything: the +flags and key (`--server` must be a plain `http` or `https` URL: no userinfo, +which would go out as Basic auth, and no query or fragment, which would swallow +the paths joined onto it), a `SILO_NODE_DRAIN_TIMEOUT` the node would ignore +(`transcodenode.ParseDrainTimeout`, which the node parses it with), FFmpeg (Homebrew's `ffmpeg-full`, as `playback`'s discovery +finds it), the server's `GET /api/v1/ready` (joined onto the URL's path, so a +server behind a path prefix works; a `200` within five seconds, no redirects, +so a server that is restarting or migrating stops the run), and the +port (nothing else may answer on it). It then takes the lock, reads the +install again under it, stages the new files beside the old ones, stops the +old job, swaps the files in, bootstraps the job, and waits up to 60 seconds +for the node's `GET /api/v1/health`. + +The stop drains before it boots the job out. `launchctl bootout` returns at +once and launchd kills the job a minute after it, whatever `ExitTimeOut` asks, +so the installer sends SIGTERM itself (`launchctl kill SIGTERM`), polls +`launchctl print` until the signaled pid is gone, and only then boots the job +out and waits for launchd to let it go. A `launchctl print` error while it +waits is retried, not taken for the exit. A job an older plist keeps alive +unconditionally restarts under a new pid; the signaled pid's exit still ends +the drain. Each wait is bounded by the running node's drain timeout plus two +minutes: the longer of the one `node.env` sets and the one the installed +plist's `ExitTimeOut` recorded, since the node read `node.env` when it started +and the file may have changed since. A node still running at that deadline is booted out anyway, and +the installer says it is stopping a node that is still finishing streams, so a +stop never leaves the job stopped without the install's restart or rollback +running. Uninstall stops the job the same way. + +The swap moves each file to `.old` and keeps it until the new node is +ready; a `.old` already there is the last working file of a run that +stopped mid-swap, so it stays the backup and the unverified file in its place +is dropped. For the same reason, an install reads its stored settings from +`node.env.old` and `node-key.old` when `node.env` or `node-key` is missing, so a +rerun after such a run keeps the previous server, port, key and other +variables instead of starting over as a first install. + +The node is ready when that answers `200` with a `node_control_version`, which +neither the API server's health nor a proxy's carries, and without +`key_rejected`. A drained node is ready: an administrator's drain outlasts the +restart, and the installer says so. The node opens its port only once the core +accepted its key, so a ready node also shows that the key and core URL work. + +When the new job does not start or become ready, or the run is interrupted +after the files were swapped, the previous files come back, with a context of +their own, and the previous job is bootstrapped only if launchd had it before +the install, so a node that was stopped stays stopped. A rollback first stops +the new job, and restores nothing until launchd confirms it is gone: when the +stop fails, every file and backup stays where it is and the installer reports +the recovery as incomplete, naming the backups. The installer exits +non-zero with the end of the log. A stop interrupted, or failed in launchctl, after its signal went out +waits until launchd lets the old job go and bootstraps its plist again, so the +old node is not left stopped. Only `launchctl print`'s "not found" status +(113) counts as a job launchd does not have. A bootout that answers +`EINPROGRESS` (36), a job still shutting down, went out, and the installer +waits for the job to go as it does after a bootout that timed out. Each +launchctl call has its own timeout. + ## Rate limit Each API replica gives each node three in-memory token buckets and answers diff --git a/go.mod b/go.mod index 5caf383b75..47ee185b85 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( golang.org/x/net v0.62.0 golang.org/x/sync v0.24.0 golang.org/x/sys v0.49.0 + golang.org/x/term v0.47.0 golang.org/x/text v0.43.0 golang.org/x/time v0.15.0 golang.org/x/tools v0.51.0 @@ -213,7 +214,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/term v0.47.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect k8s.io/klog/v2 v2.140.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/internal/fspath/launch_guard_test.go b/internal/fspath/launch_guard_test.go index 2ac69b87e0..bc0221169d 100644 --- a/internal/fspath/launch_guard_test.go +++ b/internal/fspath/launch_guard_test.go @@ -28,6 +28,7 @@ var launchesOutsideFspath = map[string]string{ "internal/backup/restore_pg.go:toc": "pg_restore --list of a dump", "internal/jellycompat/web_component.go:commandOutput": "the web client installer's git and npm", "internal/jellycompat/web_component.go:webInstallCommandRunner": "the web client installer's git and npm", + "internal/nodeinstall/launchd_darwin.go:launchctl": "launchctl on the user's own LaunchAgent", "internal/nodemetrics/nvidia.go:": "nvidia-smi", "internal/playback/gpudetect.go:runFFmpegProbe": "FFmpeg on lavfi test sources", "internal/playback/gpudetect.go:runNVIDIASMIQuery": "nvidia-smi", diff --git a/internal/imageutil/budget.go b/internal/imageutil/budget.go index a32ff7b5a5..ef54119ed7 100644 --- a/internal/imageutil/budget.go +++ b/internal/imageutil/budget.go @@ -1,8 +1,9 @@ +//go:build !silo_node + package imageutil import ( "context" - "errors" "fmt" "sync" "time" @@ -13,10 +14,6 @@ import ( "github.com/Silo-Server/silo-server/internal/nodemetrics" ) -// ErrBusy reports that a source waited maxDecodeWait for room in the decode -// budget and found none. The image may be fine; the caller can try again. -var ErrBusy = errors.New("imageutil: image processing is busy") - // maxSourcePixels is the largest source any node accepts. A file's size says // nothing about its raster: a PNG of a few hundred kilobytes can declare // billions of pixels, and bimg opens every source with random access, so diff --git a/internal/imageutil/imageutil.go b/internal/imageutil/imageutil.go index 8ee9993c3b..596755427b 100644 --- a/internal/imageutil/imageutil.go +++ b/internal/imageutil/imageutil.go @@ -1,5 +1,5 @@ -// Package imageutil provides image resizing and thumbhash generation -// for collection poster and backdrop uploads. +//go:build !silo_node + package imageutil import ( @@ -45,11 +45,6 @@ func init() { bimg.VipsCacheSetMaxMem(0) } -// ErrInvalidImage reports source bytes that libvips cannot read as an image. -// Failures after the header was read, such as encoding a variant, do not wrap -// it; see PixelDataUndecodable for damaged pixel data. -var ErrInvalidImage = errors.New("imageutil: invalid image") - // maxGoDecodePixels bounds the raster this package lets the standard library // allocate, in PixelDataUndecodable and the thumbhash fallback. A // few-kilobyte file can declare large dimensions, and image.Decode allocates @@ -89,24 +84,6 @@ func SourceUnusable(err error, data []byte) bool { return !errors.Is(err, ErrBusy) && PixelDataUndecodable(data) } -// MaxCachedOriginalDimension caps the longest edge of a cached "original" -// variant at 4K. Provider artwork is rarely larger (TMDB backdrops top out at -// 3840 wide, posters at 2000x3000), so the original keeps the provider's full -// resolution; the cap bounds an outlier upload rather than everyday art. -const MaxCachedOriginalDimension = 3840 - -// Variant holds a named image variant (e.g. "original", "w500"). -type Variant struct { - Key string - Data []byte -} - -// VariantResult contains generated variants and their output format. -type VariantResult struct { - Variants []Variant - Ext string // file extension including dot: ".webp" -} - // GenerateVariants produces WebP variants of the source image at the requested // widths, plus an "original" re-encoded as WebP. WebP provides better quality // per byte than JPEG and supports transparency (unlike JPEG). Images narrower diff --git a/internal/imageutil/imageutil_node.go b/internal/imageutil/imageutil_node.go new file mode 100644 index 0000000000..1baffc4ee2 --- /dev/null +++ b/internal/imageutil/imageutil_node.go @@ -0,0 +1,51 @@ +//go:build silo_node + +// The transcode node build (tag silo_node) links no libvips: a node never +// processes images, so every function here fails or reports false. + +package imageutil + +import "errors" + +// errNodeBuild is what every processing function returns in a node build. +var errNodeBuild = errors.New("imageutil: image processing is not part of a transcode node build") + +// NeedsRasterizing reports false in a node build. +func NeedsRasterizing([]byte) bool { return false } + +// PixelDataUndecodable reports false in a node build. +func PixelDataUndecodable([]byte) bool { return false } + +// SourceUnusable reports false in a node build. +func SourceUnusable(error, []byte) bool { return false } + +// GenerateVariants fails in a node build. +func GenerateVariants([]byte, []int) (*VariantResult, error) { return nil, errNodeBuild } + +// CheckImage fails in a node build. +func CheckImage([]byte) error { return errNodeBuild } + +// EncodeWebPWidth fails in a node build. +func EncodeWebPWidth([]byte, int) ([]byte, error) { return nil, errNodeBuild } + +// Rasterize fails in a node build. +func Rasterize([]byte) ([]byte, error) { return nil, errNodeBuild } + +// EncodePNGWithin fails in a node build. +func EncodePNGWithin([]byte, int, int) (out []byte, width, height int, err error) { + return nil, 0, 0, errNodeBuild +} + +// EncodeWebPWithin fails in a node build. +func EncodeWebPWithin([]byte, int, int) (out []byte, width, height int, err error) { + return nil, 0, 0, errNodeBuild +} + +// GenerateSquareVariants fails in a node build. +func GenerateSquareVariants([]byte, []int) (*VariantResult, error) { return nil, errNodeBuild } + +// CropToPoster fails in a node build. +func CropToPoster([]byte) ([]byte, error) { return nil, errNodeBuild } + +// Thumbhash fails in a node build. +func Thumbhash([]byte) (string, error) { return "", errNodeBuild } diff --git a/internal/imageutil/imageutil_test.go b/internal/imageutil/imageutil_test.go index 6dd58d56fd..8c2f96363e 100644 --- a/internal/imageutil/imageutil_test.go +++ b/internal/imageutil/imageutil_test.go @@ -1,3 +1,5 @@ +//go:build !silo_node + package imageutil import ( diff --git a/internal/imageutil/shared.go b/internal/imageutil/shared.go new file mode 100644 index 0000000000..24f5ca8a28 --- /dev/null +++ b/internal/imageutil/shared.go @@ -0,0 +1,36 @@ +// Package imageutil provides image resizing and thumbhash generation +// for collection poster and backdrop uploads. +package imageutil + +import "errors" + +// The declarations both builds share. The full build (imageutil.go and +// budget.go) and the transcode node build (imageutil_node.go) each define +// the functions. + +// ErrBusy reports that a source waited maxDecodeWait for room in the decode +// budget and found none. The image may be fine; the caller can try again. +var ErrBusy = errors.New("imageutil: image processing is busy") + +// ErrInvalidImage reports source bytes that libvips cannot read as an image. +// Failures after the header was read, such as encoding a variant, do not wrap +// it; see PixelDataUndecodable for damaged pixel data. +var ErrInvalidImage = errors.New("imageutil: invalid image") + +// MaxCachedOriginalDimension caps the longest edge of a cached "original" +// variant at 4K. Provider artwork is rarely larger (TMDB backdrops top out at +// 3840 wide, posters at 2000x3000), so the original keeps the provider's full +// resolution; the cap bounds an outlier upload rather than everyday art. +const MaxCachedOriginalDimension = 3840 + +// Variant holds a named image variant (e.g. "original", "w500"). +type Variant struct { + Key string + Data []byte +} + +// VariantResult contains generated variants and their output format. +type VariantResult struct { + Variants []Variant + Ext string // file extension including dot: ".webp" +} diff --git a/internal/imageutil/untrusted.go b/internal/imageutil/untrusted.go index 74fb3fec5e..7a9476ab3a 100644 --- a/internal/imageutil/untrusted.go +++ b/internal/imageutil/untrusted.go @@ -1,3 +1,5 @@ +//go:build !silo_node + package imageutil /* diff --git a/internal/imageutil/untrusted_test.go b/internal/imageutil/untrusted_test.go index 96c6df1a2d..9b01681c6d 100644 --- a/internal/imageutil/untrusted_test.go +++ b/internal/imageutil/untrusted_test.go @@ -1,3 +1,5 @@ +//go:build !silo_node + package imageutil import ( diff --git a/internal/imageutil/variants_bench_test.go b/internal/imageutil/variants_bench_test.go index f251fb1f23..b1c32a50d4 100644 --- a/internal/imageutil/variants_bench_test.go +++ b/internal/imageutil/variants_bench_test.go @@ -1,3 +1,5 @@ +//go:build !silo_node + package imageutil import ( diff --git a/internal/imageutil/variants_test.go b/internal/imageutil/variants_test.go index 64d0ceb9d5..bc4f843a8c 100644 --- a/internal/imageutil/variants_test.go +++ b/internal/imageutil/variants_test.go @@ -1,3 +1,5 @@ +//go:build !silo_node + package imageutil import ( diff --git a/internal/nodebuild/full.go b/internal/nodebuild/full.go new file mode 100644 index 0000000000..8add713f04 --- /dev/null +++ b/internal/nodebuild/full.go @@ -0,0 +1,7 @@ +//go:build !silo_node + +// Package nodebuild reports whether this binary is a transcode node build. +package nodebuild + +// Node reports a transcode node build (tag silo_node); see node.go. +const Node = false diff --git a/internal/nodebuild/node.go b/internal/nodebuild/node.go new file mode 100644 index 0000000000..f30f35cd0e --- /dev/null +++ b/internal/nodebuild/node.go @@ -0,0 +1,8 @@ +//go:build silo_node + +// Package nodebuild reports whether this binary is a transcode node build. +package nodebuild + +// Node reports a transcode node build (tag silo_node): no image +// processing and no web app, so it runs only as a transcode node. +const Node = true diff --git a/internal/nodeinstall/launchd_darwin.go b/internal/nodeinstall/launchd_darwin.go new file mode 100644 index 0000000000..03183004db --- /dev/null +++ b/internal/nodeinstall/launchd_darwin.go @@ -0,0 +1,247 @@ +//go:build darwin + +package nodeinstall + +import ( + "context" + "errors" + "fmt" + "io" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" + "time" +) + +// launchctlPath is launchctl; tests point it at a fake. +var launchctlPath = "/bin/launchctl" + +// launchctlNotFound is launchctl print's exit status for a service its domain +// does not have. Any other failure is an error, never "not loaded". +const launchctlNotFound = 113 + +// launchctlInProgress is launchctl bootout's exit status (EINPROGRESS) for a +// job that got the signal and is still shutting down. +const launchctlInProgress = 36 + +// launchctlTimeout bounds each launchctl call on its own. +const launchctlTimeout = 30 * time.Second + +// errNodeStillLoaded is a stop whose deadline passed while launchd still had +// the job, so the job is still there. +var errNodeStillLoaded = errors.New("launchd still has the node's job") + +// launchctl runs one launchctl call with its own timeout, in a process group +// of its own so a terminal's Ctrl-C does not cut it short; ctx still cancels +// it. A canceled ctx is the error then. +func (n nodeInstaller) launchctl(ctx context.Context, args ...string) ([]byte, error) { + callCtx, cancel := context.WithTimeout(ctx, launchctlTimeout) + defer cancel() + cmd := exec.CommandContext(callCtx, launchctlPath, args...) //nolint:gosec // G204: launchctl on this user's own domain, job and plist. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + output, err := cmd.CombinedOutput() + switch { + case err == nil: + return output, nil + case ctx.Err() != nil: + return output, ctx.Err() + case callCtx.Err() != nil: + // Not an exit status: launchctl gave no answer in time. + return output, fmt.Errorf("launchctl %s gave no answer within %d seconds: %w", args[0], int(launchctlTimeout/time.Second), callCtx.Err()) + default: + return output, fmt.Errorf("launchctl %s failed (%w): %s", args[0], err, strings.TrimSpace(string(output))) + } +} + +// launchdPIDLine is the running process's line in launchctl print's output. +var launchdPIDLine = regexp.MustCompile(`(?m)^\tpid = ([0-9]+)$`) + +// loaded reports whether launchd has the node's job. Only launchctl's "not +// found" status means it does not; any other failure is an error. +func (n nodeInstaller) loaded(ctx context.Context) (bool, error) { + loaded, _, err := n.status(ctx) + return loaded, err +} + +// status reports whether launchd has the node's job and, while its node runs, +// the node's pid (0 otherwise). +func (n nodeInstaller) status(ctx context.Context) (loaded bool, pid int, err error) { + output, err := n.launchctl(ctx, "print", n.service) + var exitErr *exec.ExitError + switch { + case err == nil: + if match := launchdPIDLine.FindSubmatch(output); match != nil { + pid, _ = strconv.Atoi(string(match[1])) + } + return true, pid, nil + case errors.As(err, &exitErr) && exitErr.ExitCode() == launchctlNotFound: + return false, 0, nil + default: + return false, 0, err + } +} + +// nodeStopWait bounds each of stop's waits for a node that drains for drain; +// tests shorten it. +var nodeStopWait = nodeStopTimeout + +// stop stops the job, if launchd has it, and waits until launchd no longer +// has it. launchd caps ExitTimeOut at a minute, so stop first sends the +// running node SIGTERM itself and waits for it to drain, which can take as +// long as its streams, and only then boots the job out. sent reports that a +// signal went out, so the job may be on its way down even when stop returns +// an error. +func (n nodeInstaller) stop(ctx context.Context, out io.Writer) (sent bool, err error) { + loaded, pid, err := n.status(ctx) + if err != nil || !loaded { + return false, err + } + notice := "Stopping the running node.\n" + if n.drain > 0 { + notice = "Stopping the running node. It lets active streams finish first, which can take up to " + minutes(n.drain) + ".\n" + } + if _, err := io.WriteString(out, notice); err != nil { + return false, err + } + killed := false + if pid > 0 { + if killed, err = n.drainJob(ctx, out, pid, time.Now().Add(nodeStopWait(n.drain))); err != nil { + return false, err + } + } + // The bootout is not canceled with ctx: once it starts it may have gone + // out, and its own timeout bounds it. + _, bootoutErr := n.launchctl(context.WithoutCancel(ctx), "bootout", n.service) + var exitErr *exec.ExitError + if errors.As(bootoutErr, &exitErr) && exitErr.ExitCode() != launchctlInProgress { + // launchctl refused it: the job is gone, or it was left as it is. + loaded, err := n.loaded(ctx) + if err == nil && !loaded { + return true, nil + } + if err == nil && killed { + // The job stays, but its node may have drained and exited, which + // KeepAlive does not restart: start it again. + _, err = n.launchctl(context.WithoutCancel(ctx), "kickstart", n.service) + } + return false, errors.Join(bootoutErr, err) + } + // Sent (the job may still be shutting down), or still waiting for an + // answer when its timeout cut it off. + if err := n.waitGone(ctx, time.Now().Add(nodeStopWait(n.drain))); err != nil { + if errors.Is(err, errNodeStillLoaded) { + err = fmt.Errorf("the running node did not stop within %s: %w", minutes(nodeStopWait(n.drain)), err) + } + return true, err + } + return true, nil +} + +// drainJob sends the node running as pid SIGTERM through launchd and waits, +// until deadline, for that process to exit. A job an older plist keeps alive +// may start again under a new pid; the signaled pid's exit still ends the +// drain. killed reports that the signal went out. A launchctl error while +// polling is retried until the deadline, since bootout would cut the drain +// short. Once the signal went out, every way out (the exit, the deadline, an +// interrupt) leads to the bootout, so the job is never left stopped by a drained node alone; an +// error means ctx ended before anything was sent. +func (n nodeInstaller) drainJob(ctx context.Context, out io.Writer, pid int, deadline time.Time) (killed bool, err error) { + if err := ctx.Err(); err != nil { + return false, err + } + // Not canceled with ctx, as the bootout: once it starts it may have gone out. + if _, err := n.launchctl(context.WithoutCancel(ctx), "kill", "SIGTERM", n.service); err != nil { + // No signal: the bootout sends one. + return false, nil + } + ticker := time.NewTicker(nodePollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return true, nil + case <-ticker.C: + } + loaded, running, err := n.status(ctx) + switch { + case err == nil && (!loaded || running != pid): + return true, nil + case !time.Now().Before(deadline): + _, _ = fmt.Fprintf(out, "The node is still finishing streams after %s and is being stopped now.\n", minutes(nodeStopWait(n.drain))) + return true, nil + } + } +} + +// waitGone polls launchd until it no longer has the job. At deadline it +// reports errNodeStillLoaded. +func (n nodeInstaller) waitGone(ctx context.Context, deadline time.Time) error { + ticker := time.NewTicker(nodePollInterval) + defer ticker.Stop() + for { + loaded, err := n.loaded(ctx) + switch { + case err != nil: + return err + case !loaded: + return nil + case !time.Now().Before(deadline): + return errNodeStillLoaded + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// minutes names d in whole minutes, rounded up. +func minutes(d time.Duration) string { + m := (d + time.Minute - 1) / time.Minute + if m == 1 { + return "1 minute" + } + return strconv.FormatInt(int64(m), 10) + " minutes" +} + +// stopRunning stops the job for an install or uninstall, and reports whether +// launchd had it. When the stop ends early after the bootout went out, +// interrupted or on a launchctl error, the job is started again from the +// plist still in place, so the node is not left stopped. +func (n nodeInstaller) stopRunning(ctx context.Context, hasPlist bool) (wasLoaded bool, err error) { + sent, err := n.stop(ctx, n.out) + if err == nil { + return sent, nil + } + stillLoaded := errors.Is(err, errNodeStillLoaded) + if ctx.Err() != nil { + err = errors.New("interrupted") + } + if !sent || !hasPlist || stillLoaded { + return sent, err + } + if ctx.Err() != nil { + _, _ = fmt.Fprintln(n.out, "Interrupted: the previous node starts again once it has stopped. "+ + "Interrupt again to leave it stopped until the next login.") + } + restartCtx, cancel := context.WithTimeout(context.Background(), nodeStopTimeout(n.drain)) + defer cancel() + restartErr := n.waitGone(restartCtx, time.Now().Add(nodeStopTimeout(n.drain))) + if restartErr == nil { + restartErr = n.bootstrap(restartCtx) + } + if restartErr != nil { + return sent, fmt.Errorf("%w\n%s", err, "Starting the previous node again failed: "+restartErr.Error()) + } + return sent, fmt.Errorf("%w\n%s", err, "The previous node is running again.") +} + +func (n nodeInstaller) bootstrap(ctx context.Context) error { + if _, err := n.launchctl(ctx, "bootstrap", n.domain, n.paths.plist); err != nil { + return fmt.Errorf("%w; the node needs a logged-in user session", err) + } + return nil +} diff --git a/internal/nodeinstall/node_command.go b/internal/nodeinstall/node_command.go new file mode 100644 index 0000000000..c9d78b880e --- /dev/null +++ b/internal/nodeinstall/node_command.go @@ -0,0 +1,313 @@ +// Package nodeinstall is the "silo node" command, which installs a Mac's +// transcode node as a LaunchAgent. +package nodeinstall + +import ( + "bytes" + "encoding/xml" + "errors" + "flag" + "fmt" + "io" + "maps" + "math" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + "unicode" + + "github.com/joho/godotenv" + + "github.com/Silo-Server/silo-server/internal/buildinfo" + "github.com/Silo-Server/silo-server/internal/envutil" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +const nodeCommandUsage = `usage: + silo node install [--server URL] [--new-key] [--port N] + Install this Mac's transcode node as a LaunchAgent, or upgrade it to this binary + silo node uninstall + Stop the node and remove it, keeping its log + silo node version + Print this build's revision and build number` + +const nodeInstallUsage = `usage: silo node install [--server URL] [--new-key] [--port N] + +Installs this Mac's transcode node as a LaunchAgent for the current user, or +upgrades it to this binary. A rerun keeps every value it is not given, and any +other variable added to node.env, so "silo node install" alone upgrades in +place. + +The node key Admin > Nodes issued is asked for at a hidden prompt the first +time and with --new-key. To script an install, pipe the key in on standard +input instead. It is never taken from the command line, where other users of +this Mac could read it. + + --server URL the Silo server's URL as this Mac reaches it, such as + http://localhost:8090 (required the first time) + --new-key replace the installed node key + --port N the port the node listens on (default 8100)` + +// nodeLaunchdLabel names the node's LaunchAgent. +const nodeLaunchdLabel = "org.siloserver.node" + +// defaultNodeInstallPort is the installed node's port. A node otherwise +// listens on 8080, which Docker Desktop often publishes already. +const defaultNodeInstallPort = 8100 + +// nodeKeyPrompt marks a run that asks for a new key (--new-key). +const nodeKeyPrompt = "-" + +// nodeExitTimeout is how long launchd should wait after SIGTERM before it kills +// a node that drains for drain: the drain and a minute to shut down. Current +// macOS caps ExitTimeOut at 60 seconds, so the installer sends SIGTERM itself +// and waits for the drain before its bootout. +func nodeExitTimeout(drain time.Duration) time.Duration { + return addCapped(drain, time.Minute) +} + +func addCapped(d, extra time.Duration) time.Duration { + if d > math.MaxInt64-extra { + return math.MaxInt64 + } + return d + extra +} + +// nodeEnvDrainTimeout is the drain the node takes from node.env's other +// variables. A value the node would not accept is an error. +func nodeEnvDrainTimeout(extra map[string]string) (time.Duration, error) { + drain, err := transcodenode.ParseDrainTimeout(extra[transcodenode.EnvDrainTimeout]) + if err != nil { + return 0, fmt.Errorf("node.env: %w", err) + } + return drain, nil +} + +// Run serves "silo node install|uninstall|version". +func Run(args []string, in io.Reader, out io.Writer) error { + if len(args) == 0 { + return errors.New(nodeCommandUsage) + } + switch args[0] { + case "install": + return installNode(args[1:], in, out) + case "uninstall": + if len(args) != 1 { + return errors.New(nodeCommandUsage) + } + return uninstallNode(out) + case "version": + if len(args) != 1 { + return errors.New(nodeCommandUsage) + } + _, err := fmt.Fprintln(out, nodeVersion(buildinfo.Current())) + return err + case "-h", "-help", "--help", "help": + _, err := fmt.Fprintln(out, nodeCommandUsage) + return err + default: + return errors.New(nodeCommandUsage) + } +} + +// nodeVersion names a build as Admin shows the server's and each node's: the +// build number and the display revision, or the revision alone. +func nodeVersion(info buildinfo.Info) string { + if info.BuildNumber == 0 { + return info.Display + } + return strconv.FormatUint(info.BuildNumber, 10) + " ยท " + info.Display +} + +// nodeInstallSettings is what an install sets. A zero field is one not given +// or not stored. +type nodeInstallSettings struct { + server string + key string + port int +} + +// parseNodeInstallFlags reads install's flags. help reports -h or --help. +func parseNodeInstallFlags(args []string) (given nodeInstallSettings, help bool, err error) { + flags := flag.NewFlagSet("silo node install", flag.ContinueOnError) + flags.SetOutput(io.Discard) + flags.StringVar(&given.server, "server", "", "") + newKey := flags.Bool("new-key", false, "") + flags.IntVar(&given.port, "port", 0, "") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nodeInstallSettings{}, true, nil + } + return nodeInstallSettings{}, false, fmt.Errorf("%w\n%s", err, nodeInstallUsage) + } + if flags.NArg() != 0 { + return nodeInstallSettings{}, false, errors.New(nodeInstallUsage) + } + given.server = strings.TrimSpace(given.server) + if *newKey { + given.key = nodeKeyPrompt + } + // A flag given is checked by its parsed value, so --port 00 is no port + // and --server "" no server. + flags.Visit(func(f *flag.Flag) { + switch { + case err != nil: + case f.Name == "port" && (given.port < 1 || given.port > 65535): + err = errors.New("--port must be between 1 and 65535") + case f.Name == "server" && given.server == "": + err = errors.New("--server needs a value") + } + }) + return given, false, err +} + +// mergeNodeInstall lays the flags given over the existing install's values +// and fills in the default port. +func mergeNodeInstall(stored, given nodeInstallSettings) nodeInstallSettings { + merged := stored + if given.server != "" { + merged.server = given.server + } + if given.key != "" { + merged.key = given.key + } + if given.port != 0 { + merged.port = given.port + } + if merged.port == 0 { + merged.port = defaultNodeInstallPort + } + return merged +} + +// nodeEnvManaged are the node.env variables the installer writes. A node key +// set directly would clash with the key file, so it is dropped as well. +var nodeEnvManaged = []string{"MODE", transcodenode.EnvCoreURL, transcodenode.EnvNodeKey + envutil.SecretFileSuffix, "PORT", transcodenode.EnvNodeKey} + +// storedNodeInstall reads an existing install's settings from its node.env +// and node-key contents. extra holds the other variables in node.env, which +// a rerun keeps. +func storedNodeInstall(env, key []byte) (stored nodeInstallSettings, extra map[string]string, err error) { + values, err := godotenv.Unmarshal(string(env)) + if err != nil { + return nodeInstallSettings{}, nil, fmt.Errorf("node.env: %w", err) + } + stored = nodeInstallSettings{server: strings.TrimSpace(values[transcodenode.EnvCoreURL]), key: strings.TrimSpace(string(key))} + if port, err := strconv.Atoi(strings.TrimSpace(values["PORT"])); err == nil { + stored.port = port + } + for _, name := range nodeEnvManaged { + delete(values, name) + } + return stored, values, nil +} + +// nodeInstallPaths are an install's files for the user whose home is home. +type nodeInstallPaths struct { + dir string + binary string + env string + key string + lock string + log string + plist string +} + +func nodeInstallPathsIn(home string) nodeInstallPaths { + dir := filepath.Join(home, "Library", "Application Support", "Silo", "node") + return nodeInstallPaths{ + dir: dir, + binary: filepath.Join(dir, "silo"), + env: filepath.Join(dir, "node.env"), + key: filepath.Join(dir, "node-key"), + lock: filepath.Join(filepath.Dir(dir), "node-install.lock"), // outside dir, which uninstall removes under the lock + log: filepath.Join(home, "Library", "Logs", "Silo", "node.log"), + plist: filepath.Join(home, "Library", "LaunchAgents", nodeLaunchdLabel+".plist"), + } +} + +// renderNodeEnv writes node.env: the mode, the server's URL, the key file and +// the port, then the other variables an existing install carried, in name +// order. The node takes everything else from the server. +func renderNodeEnv(s nodeInstallSettings, keyFile string, extra map[string]string) ([]byte, error) { + entries := [][2]string{ + {"MODE", transcodenode.Mode}, + {transcodenode.EnvCoreURL, s.server}, + {transcodenode.EnvNodeKey + envutil.SecretFileSuffix, keyFile}, + {"PORT", strconv.Itoa(s.port)}, + } + for _, name := range slices.Sorted(maps.Keys(extra)) { + entries = append(entries, [2]string{name, extra[name]}) + } + var b strings.Builder + for _, entry := range entries { + value, err := quoteNodeEnv(entry[1]) + if err != nil { + return nil, fmt.Errorf("%s: %w", entry[0], err) + } + fmt.Fprintf(&b, "%s=%s\n", entry[0], value) + } + return []byte(b.String()), nil +} + +// quoteNodeEnv quotes value for godotenv, which the node reads node.env with: +// in double quotes, with $ escaped so it is not expanded, or in single quotes, +// which expand nothing, when it holds a double quote. godotenv cannot read +// back every backslash, nor a value with both quotes, so those are refused, +// with control characters. +func quoteNodeEnv(value string) (string, error) { + if strings.Contains(value, `\`) || strings.ContainsFunc(value, unicode.IsControl) || + (strings.Contains(value, `"`) && strings.Contains(value, "'")) { + return "", errors.New("the value holds a backslash, a control character, or both kinds of quote, which node.env cannot carry") + } + if strings.Contains(value, `"`) { + return "'" + value + "'", nil + } + return `"` + strings.ReplaceAll(value, "$", `\$`) + `"`, nil +} + +// renderNodePlist writes the LaunchAgent that runs the installed node, which +// drains for drain on SIGTERM. +func renderNodePlist(p nodeInstallPaths, drain time.Duration) []byte { + var b bytes.Buffer + str := func(s string) { + b.WriteString("") + _ = xml.EscapeText(&b, []byte(s)) + b.WriteString("\n") + } + key := func(k string) { b.WriteString("\t" + k + "\n\t") } + b.WriteString(` + + + +`) + key("Label") + str(nodeLaunchdLabel) + key("ProgramArguments") + b.WriteString("\n") + for _, arg := range []string{p.binary, "-env", p.env} { + b.WriteString("\t\t") + str(arg) + } + b.WriteString("\t\n") + key("WorkingDirectory") + str(p.dir) + key("StandardOutPath") + str(p.log) + key("StandardErrorPath") + str(p.log) + key("RunAtLoad") + b.WriteString("\n") + // Restarted after a crash or a failed exit, but not after a drained node's + // clean exit, which the installer's stop waits for. + key("KeepAlive") + b.WriteString("\n\t\tSuccessfulExit\n\t\t\n\t\n") + key("ProcessType") + str("Interactive") + key("ExitTimeOut") + fmt.Fprintf(&b, "%d\n", int(nodeExitTimeout(drain)/time.Second)) + b.WriteString("\n\n") + return b.Bytes() +} diff --git a/internal/nodeinstall/node_command_test.go b/internal/nodeinstall/node_command_test.go new file mode 100644 index 0000000000..4244ec1bc4 --- /dev/null +++ b/internal/nodeinstall/node_command_test.go @@ -0,0 +1,194 @@ +package nodeinstall + +import ( + "encoding/xml" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/joho/godotenv" + + "github.com/Silo-Server/silo-server/internal/envutil" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// The node.env the installer writes is one the node starts from, with the key +// read from node-key, even under a home whose path needs quoting. +func TestRenderNodeEnvStartsTheNode(t *testing.T) { + key, err := nodepool.GenerateNodeKey(6) + if err != nil { + t.Fatal(err) + } + paths := nodeInstallPathsIn(filepath.Join(t.TempDir(), "home $USER #1")) + if err := os.MkdirAll(paths.dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(paths.key, []byte(key+"\n"), 0o600); err != nil { + t.Fatal(err) + } + settings := nodeInstallSettings{server: "http://localhost:8090", key: key, port: 8100} + env, err := renderNodeEnv(settings, paths.key, nil) + if err != nil { + t.Fatal(err) + } + values, err := godotenv.Unmarshal(string(env)) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"MODE": transcodenode.Mode, transcodenode.EnvCoreURL: settings.server, "SILO_NODE_KEY_FILE": paths.key, "PORT": "8100"} + if len(values) != len(want) { + t.Fatalf("node.env = %v, want exactly %v", values, want) + } + for name, value := range want { + if values[name] != value { + t.Fatalf("%s = %q, want %q", name, values[name], value) + } + } + // The node reads its key as envutil.LookupSecret finds it, from node-key. + gotKey, _, err := envutil.LookupSecret(func(name string) string { return values[name] }, transcodenode.EnvNodeKey) + if err != nil || strings.TrimSpace(gotKey) != key { + t.Fatalf("node key = %q, %v; want the installed key", gotKey, err) + } + if _, err := nodepool.ParseNodeKey(strings.TrimSpace(gotKey)); err != nil { + t.Fatalf("installed key: %v", err) + } + if _, ok := transcodenode.ParseCoreURL(values[transcodenode.EnvCoreURL]); !ok { + t.Fatalf("%s = %q, not a URL the node accepts", transcodenode.EnvCoreURL, values[transcodenode.EnvCoreURL]) + } + + // A rerun reads the same settings back, and keeps a variable an operator + // added to node.env. + added := map[string]string{"SILO_NODE_DRAIN_TIMEOUT": "15m"} + stored, extra, err := storedNodeInstall(append(env, "SILO_NODE_DRAIN_TIMEOUT=15m\n"...), []byte(key+"\n")) + if err != nil || stored != settings || !maps.Equal(extra, added) { + t.Fatalf("stored = %+v, %v, %v; want %+v and %v", stored, extra, err, settings, added) + } + rerun, err := renderNodeEnv(stored, paths.key, extra) + if err != nil { + t.Fatal(err) + } + if values, err := godotenv.Unmarshal(string(rerun)); err != nil || values["SILO_NODE_DRAIN_TIMEOUT"] != "15m" || len(values) != len(want)+1 { + t.Fatalf("rerun node.env = %v, %v; want the managed variables and the added one", values, err) + } +} + +func TestMergeNodeInstallFlags(t *testing.T) { + stored := nodeInstallSettings{server: "http://localhost:8090", key: "stored-key", port: 8200} + tests := []struct { + name string + stored nodeInstallSettings + args []string + want nodeInstallSettings + }{ + {"a rerun without flags keeps the install", stored, nil, stored}, + {"flags override", stored, []string{"--server", "https://silo.example", "--port", "9000"}, + nodeInstallSettings{server: "https://silo.example", key: "stored-key", port: 9000}}, + {"a first install gets the default port", nodeInstallSettings{}, []string{"--server", "http://localhost:8090"}, + nodeInstallSettings{server: "http://localhost:8090", port: defaultNodeInstallPort}}, + {"--new-key asks for a new key", stored, []string{"--new-key"}, + nodeInstallSettings{server: "http://localhost:8090", key: nodeKeyPrompt, port: 8200}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + given, help, err := parseNodeInstallFlags(tt.args) + if err != nil || help { + t.Fatalf("parse = %v, help %v", err, help) + } + if got := mergeNodeInstall(tt.stored, given); got != tt.want { + t.Fatalf("merged = %+v, want %+v", got, tt.want) + } + }) + } +} + +// The plist's paths survive XML escaping, and launchd waits for the drain +// node.env sets. +func TestRenderNodePlistEscapesPaths(t *testing.T) { + paths := nodeInstallPathsIn("/Users/a & ") + var doc struct { + Dict struct { + Items []struct { + XMLName xml.Name + Value string `xml:",chardata"` + Strings []string `xml:"string"` + } `xml:",any"` + } `xml:"dict"` + } + if err := xml.Unmarshal(renderNodePlist(paths, 15*time.Minute), &doc); err != nil { + t.Fatal(err) + } + values := map[string][]string{} + items := doc.Dict.Items + for i := 0; i+1 < len(items); i += 2 { + value := items[i+1].Strings + if items[i+1].XMLName.Local != "array" { + value = []string{items[i+1].Value} + } + values[items[i].Value] = value + } + if got := values["ProgramArguments"]; !slices.Equal(got, []string{paths.binary, "-env", paths.env}) { + t.Fatalf("ProgramArguments = %q", got) + } + if got := values["StandardErrorPath"]; !slices.Equal(got, []string{paths.log}) { + t.Fatalf("StandardErrorPath = %q, want %q", got, paths.log) + } + if got := values["ExitTimeOut"]; !slices.Equal(got, []string{"960"}) { + t.Fatalf("ExitTimeOut = %q, want 960 (a 15-minute drain and a minute)", got) + } +} + +// The installer takes node.env's drain as the node does, and refuses a value +// the node would ignore. +func TestNodeEnvDrainTimeout(t *testing.T) { + for _, test := range []struct { + value string + want time.Duration + ok bool + }{ + {"", transcodenode.DefaultDrainTimeout, true}, + {"0", 0, true}, + {"900", 15 * time.Minute, true}, + {" 2h ", 2 * time.Hour, true}, + {"-5m", 0, false}, + {"soon", 0, false}, + } { + extra := map[string]string{} + if test.value != "" { + extra[transcodenode.EnvDrainTimeout] = test.value + } + got, err := nodeEnvDrainTimeout(extra) + if got != test.want || (err == nil) != test.ok { + t.Errorf("nodeEnvDrainTimeout(%q) = %v, %v; want %v, ok %v", test.value, got, err, test.want, test.ok) + } + } +} + +// A kept node.env value with a double quote is written so the node reads it +// back unchanged; one that cannot round-trip is refused. +func TestRenderNodeEnvKeepsQuotedValues(t *testing.T) { + settings := nodeInstallSettings{server: "http://localhost:8090", key: "k", port: 8100} + extra := map[string]string{"SILO_NODE_NAME": `Mac "Studio"`, "OTHER": "it's $HOME"} + env, err := renderNodeEnv(settings, "/k", extra) + if err != nil { + t.Fatal(err) + } + values, err := godotenv.Unmarshal(string(env)) + if err != nil { + t.Fatal(err) + } + for name, value := range extra { + if values[name] != value { + t.Fatalf("%s = %q, want %q", name, values[name], value) + } + } + for _, value := range []string{`both " and '`, `back\slash`, "new\nline"} { + if _, err := renderNodeEnv(settings, "/k", map[string]string{"X": value}); err == nil { + t.Errorf("renderNodeEnv kept %q, which node.env cannot carry", value) + } + } +} diff --git a/internal/nodeinstall/node_install_darwin.go b/internal/nodeinstall/node_install_darwin.go new file mode 100644 index 0000000000..5a9ebe8d74 --- /dev/null +++ b/internal/nodeinstall/node_install_darwin.go @@ -0,0 +1,829 @@ +//go:build darwin + +package nodeinstall + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" + "golang.org/x/term" + + "github.com/Silo-Server/silo-server/internal/buildinfo" + "github.com/Silo-Server/silo-server/internal/nodebuild" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// nodeServerReadyTimeout bounds the server's readiness probe before an install. +const nodeServerReadyTimeout = 5 * time.Second + +// nodeReadyTimeout bounds the wait for a new node's health answer. The node +// opens its port only once the server took its key. +const nodeReadyTimeout = 60 * time.Second + +// nodePollInterval is how often the installer looks at launchd and health. +const nodePollInterval = 500 * time.Millisecond + +// nodeImageCopyHint is how to get the node binary that matches the server. +const nodeImageCopyHint = "docker compose cp silo:/usr/share/silo/nodes/silo-darwin-arm64 ./silo" + +// nodeInstaller installs, upgrades and removes the per-user LaunchAgent. +type nodeInstaller struct { + paths nodeInstallPaths + service string // gui//org.siloserver.node + domain string // gui/ + out io.Writer + // drain is how long the running node may drain on SIGTERM, which bounds + // every wait for it to stop: the longer of node.env's drain and the one its + // plist recorded when it was installed, since the node read node.env when + // it started and the file may have changed since. + drain time.Duration + // newDrain is the drain the node.env being installed sets, which the new + // plist records. + newDrain time.Duration +} + +func newNodeInstaller(out io.Writer) (nodeInstaller, error) { + if os.Geteuid() == 0 { + return nodeInstaller{}, errors.New("run this as your own user, without sudo: the node is a LaunchAgent for that user") + } + home, err := os.UserHomeDir() + if err != nil { + return nodeInstaller{}, err + } + domain := "gui/" + strconv.Itoa(os.Getuid()) + return nodeInstaller{paths: nodeInstallPathsIn(home), domain: domain, service: domain + "/" + nodeLaunchdLabel, out: out, + drain: transcodenode.DefaultDrainTimeout}, nil +} + +// nodeSignalContext ends at SIGINT or SIGTERM. After the first signal the +// default handling is back, so a second one ends the process at once. +func nodeSignalContext() (context.Context, context.CancelFunc) { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + context.AfterFunc(ctx, stop) + return ctx, stop +} + +func installNode(args []string, in io.Reader, out io.Writer) error { + given, help, err := parseNodeInstallFlags(args) + if err != nil { + return err + } + if help { + _, err := fmt.Fprintln(out, nodeInstallUsage) + return err + } + if !nodebuild.Node { + return errors.New("this silo build links Homebrew libraries, so a Homebrew upgrade can stop it from starting. " + + "Install the node binary from the server image instead:\n " + nodeImageCopyHint + "\n ./silo node install --server ") + } + n, err := newNodeInstaller(out) + if err != nil { + return err + } + // Everything is checked, and the key asked for once, before anything is + // written or locked, so a refused run leaves nothing behind. + stored, extra, err := n.stored() + if err != nil { + return err + } + if given.key, err = askNodeKey(stored, given, func() (string, error) { return promptNodeKey(in, out) }); err != nil { + return err + } + settings, err := resolveNodeInstall(stored, given) + if err != nil { + return err + } + if _, err := nodeEnvDrainTimeout(extra); err != nil { + return err + } + if !playback.HomebrewFFmpegFull(playback.ResolveFFmpegPath("")) { + return errors.New("ffmpeg-full from Homebrew is not installed: run `brew install ffmpeg-full`, then run this again") + } + ctx, stopSignals := nodeSignalContext() + defer stopSignals() + if err := checkServerReady(ctx, settings.server); err != nil { + return err + } + unlock, err := lockNodeInstall(n.paths.lock) + if err != nil { + return err + } + defer unlock() + if err := os.MkdirAll(n.paths.dir, 0o700); err != nil { + return err + } + return n.installLocked(ctx, given, settings.server) +} + +// askNodeKey returns the key to give the install: none when the install has one +// to keep, or the one entered at the prompt (or piped in on standard input) +// when --new-key asks for it or the install has none. +func askNodeKey(stored, given nodeInstallSettings, prompt func() (string, error)) (string, error) { + asked := given.key == nodeKeyPrompt + merged := mergeNodeInstall(stored, given) + if !asked && merged.key != "" { + return given.key, nil + } + if err := merged.validateAllButKey(); err != nil { + return "", err + } + key, err := prompt() + if err == nil && asked && key == "" { + err = errors.New("--new-key needs the key: enter it at the prompt, or pipe it in on standard input") + } + return key, err +} + +// installLocked reads the install again under the lock, since another run may +// have changed it, and installs. +func (n nodeInstaller) installLocked(ctx context.Context, given nodeInstallSettings, checkedServer string) error { + stored, extra, err := n.stored() + if err != nil { + return err + } + settings, err := resolveNodeInstall(stored, given) + if err != nil { + return err + } + n.dropStaleBackups() + // The new node.env keeps the drain, so it sets the new plist's ExitTimeOut + // and, with the drain the running node's plist recorded, bounds its stop. + if n.newDrain, err = nodeEnvDrainTimeout(extra); err != nil { + return err + } + n.drain = max(n.newDrain, n.installedPlistDrain()) + if settings.server != checkedServer { + if err := checkServerReady(ctx, settings.server); err != nil { + return err + } + } + loaded, err := n.loaded(ctx) + if err != nil { + return err + } + if ours := loaded && stored.port == settings.port; !ours && nodePortTaken(ctx, settings.port) { + return fmt.Errorf("another program listens on port %d; choose a free one with --port", settings.port) + } + return n.install(ctx, settings, stored.port, extra) +} + +// resolveNodeInstall lays the flags given over the stored install and +// validates the result. +func resolveNodeInstall(stored, given nodeInstallSettings) (nodeInstallSettings, error) { + settings := mergeNodeInstall(stored, given) + return settings, settings.validate() +} + +// validate refuses settings the node would refuse at startup. +func (s nodeInstallSettings) validate() error { + if err := s.validateAllButKey(); err != nil { + return err + } + if s.key == "" { + return errors.New("the node key is required the first time: enter the key Admin > Nodes issued at the prompt, or pipe it in on standard input") + } + if _, err := nodepool.ParseNodeKey(s.key); err != nil { + return fmt.Errorf("the node key is malformed (%w); copy the key Admin > Nodes issued", err) + } + return nil +} + +func (s nodeInstallSettings) validateAllButKey() error { + if s.server == "" { + return errors.New("--server is required the first time: the Silo server's URL as this Mac reaches it, such as http://localhost:8090") + } + parsed, ok := transcodenode.ParseCoreURL(s.server) + if !ok { + return errors.New("--server must be an absolute http or https URL") + } + // The readiness probe and the node join paths onto it, which a query or + // fragment would swallow, and userinfo would go out as Basic auth. + if parsed.User != nil || strings.ContainsAny(s.server, "?#") { + return errors.New("--server must be a plain http or https URL such as http://localhost:8090") + } + if _, err := quoteNodeEnv(s.server); err != nil { + return fmt.Errorf("--server: %w", err) + } + if s.port < 1 || s.port > 65535 { + return errors.New("--port must be between 1 and 65535") + } + return nil +} + +// checkServerReady asks the server's readiness probe before anything +// changes, so a run while the server restarts or migrates stops here. +func checkServerReady(ctx context.Context, server string) error { + ctx, cancel := context.WithTimeout(ctx, nodeServerReadyTimeout) + defer cancel() + client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + reason := "" + readyURL, err := nodeServerReadyURL(server) + var req *http.Request + if err == nil { + req, err = http.NewRequestWithContext(ctx, http.MethodGet, readyURL, nil) + } + if err == nil { + var resp *http.Response + if resp, err = client.Do(req); err == nil { + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + reason = "its readiness check answered " + resp.Status + } + } + var urlErr *url.Error + switch { + case reason != "": + case errors.Is(err, context.Canceled): + return errors.New("interrupted") + case errors.Is(err, context.DeadlineExceeded): + reason = fmt.Sprintf("no answer within %d seconds", int(nodeServerReadyTimeout/time.Second)) + case errors.As(err, &urlErr): + reason = urlErr.Err.Error() + default: + reason = err.Error() + } + return fmt.Errorf("the server at %s isn't ready yet (%s); run this again once it is", server, reason) +} + +// nodeServerReadyURL is the readiness probe under server, a URL validate +// accepted, with any path it has kept as a prefix. +func nodeServerReadyURL(server string) (string, error) { + parsed, err := url.Parse(server) + if err != nil { + return "", err + } + return parsed.JoinPath("api", "v1", "ready").String(), nil +} + +// stored reads the existing install's settings, all zero without one, and the +// other variables its node.env carries. A run stopped mid-swap can leave a +// file only at its backup, .old, which is then the one read. +func (n nodeInstaller) stored() (nodeInstallSettings, map[string]string, error) { + env, err := readInstalledFile(n.paths.env) + if errors.Is(err, fs.ErrNotExist) { + return nodeInstallSettings{}, nil, nil + } + if err != nil { + return nodeInstallSettings{}, nil, err + } + key, err := readInstalledFile(n.paths.key) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nodeInstallSettings{}, nil, err + } + return storedNodeInstall(env, key) +} + +// readInstalledFile reads path, or its backup when path is missing. +// nodeStopTimeout bounds the installer's wait for a node that drains for +// drain to exit: launchd's ExitTimeOut and a minute more. +func nodeStopTimeout(drain time.Duration) time.Duration { + return addCapped(drain, 2*time.Minute) +} + +// plistExitTimeOut finds the ExitTimeOut renderNodePlist writes. +var plistExitTimeOut = regexp.MustCompile(`ExitTimeOut\s*([0-9]+)`) + +// installedPlistDrain is the drain the installed plist recorded: its +// ExitTimeOut less the minute nodeExitTimeout adds. 0 without a plist or one +// it can't read. +func (n nodeInstaller) installedPlistDrain() time.Duration { + plist, err := os.ReadFile(n.paths.plist) + if err != nil { + return 0 + } + match := plistExitTimeOut.FindSubmatch(plist) + if match == nil { + return 0 + } + seconds, err := strconv.Atoi(string(match[1])) + if err != nil || seconds <= 60 { + return 0 + } + return time.Duration(seconds-60) * time.Second +} + +// swapPending is the marker an install writes before it moves files aside and +// removes once the new node is ready. With the marker, a .old file is the last +// working install; without it, a leftover of a run that finished. +func (p nodeInstallPaths) swapPending() string { return filepath.Join(p.dir, ".swap-pending") } + +// installedFiles are the files a swap replaces. +func (p nodeInstallPaths) installedFiles() []string { return []string{p.binary, p.env, p.key} } + +// dropStaleBackups removes the .old files a finished run left behind, so a +// later rollback can't restore an older install than the one that ran. +func (n nodeInstaller) dropStaleBackups() { + if _, err := os.Stat(n.paths.swapPending()); err == nil { + return + } + for _, path := range n.paths.installedFiles() { + _ = os.Remove(path + ".old") + } +} + +func readInstalledFile(path string) ([]byte, error) { + data, err := os.ReadFile(path) //nolint:gosec // G304: the installer's own files. + if errors.Is(err, fs.ErrNotExist) { + data, err = os.ReadFile(nodeInstallSwap{path: path}.old()) + } + return data, err +} + +// install stages the new binary and settings beside the old ones, stops the +// old job, swaps the files in and starts the new job. When the new job does +// not start or become ready, or the run is interrupted, the previous files +// and job come back. +func (n nodeInstaller) install(ctx context.Context, settings nodeInstallSettings, oldPort int, extra map[string]string) error { + env, err := renderNodeEnv(settings, n.paths.key, extra) + if err != nil { + return err + } + for _, file := range []string{n.paths.log, n.paths.plist} { + if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { + return err + } + } + swaps, err := n.stage(env, settings.key) + if err != nil { + discardStaged(swaps) + return err + } + oldPlist, err := os.ReadFile(n.paths.plist) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + discardStaged(swaps) + return err + } + oldLoaded, err := n.stopRunning(ctx, oldPlist != nil) + if err != nil { + discardStaged(swaps) + return err + } + logStart := fileSize(n.paths.log) + err = writeNodeFile(n.paths.swapPending(), nil, 0o600) + if err == nil { + err = swapIn(swaps) + } + if err == nil { + err = writeNodeFile(n.paths.plist, renderNodePlist(n.paths, n.newDrain), 0o644) + } + if err == nil { + err = n.bootstrap(ctx) + } + drained := false + if err == nil { + drained, err = waitNodeReady(ctx, settings) + } + if err != nil { + return n.failInstall(ctx, err, swaps, oldPlist, oldLoaded, logStart) + } + // The swap is committed once the marker is gone; every backup goes after. + _ = os.Remove(n.paths.swapPending()) + for _, path := range n.paths.installedFiles() { + _ = os.Remove(path + ".old") + } + return n.reportInstalled(settings, oldPort, drained) +} + +// failInstall rolls back an install whose new job did not come up, with a +// context of its own so an interrupted run still puts the previous node back. +// oldLoaded reports whether launchd had the previous job before the install. +func (n nodeInstaller) failInstall(ctx context.Context, err error, swaps []nodeInstallSwap, oldPlist []byte, oldLoaded bool, logStart int64) error { + if ctx.Err() != nil { + err = errors.New("interrupted") + _, _ = fmt.Fprintln(n.out, "Interrupted: putting the previous node back. Interrupt again to stop at once.") + } + tail := nodeLogTail(n.paths.log, logStart) + restoreCtx, cancel := context.WithTimeout(context.Background(), nodeStopTimeout(n.drain)) + defer cancel() + restored := "The previous node was restored." + if rollbackErr := n.rollback(restoreCtx, swaps, oldPlist, oldLoaded); rollbackErr != nil { + restored = "Restoring the previous node failed: " + rollbackErr.Error() + } else if oldPlist == nil { + restored = "Nothing was installed." + } + return fmt.Errorf("%w\n%s\nLast lines of %s:\n%s", err, restored, n.paths.log, tail) +} + +func (n nodeInstaller) reportInstalled(settings nodeInstallSettings, oldPort int, drained bool) error { + var b strings.Builder + fmt.Fprintf(&b, "Installed the Silo transcode node (build %s).\nLog: %s\n", nodeVersion(buildinfo.Current()), n.paths.log) + if drained { + b.WriteString("The node is drained: an administrator resumes it in Admin > Nodes.\n") + } + if oldPort != 0 && oldPort != settings.port { + fmt.Fprintf(&b, "The port changed from %d to %d, so the node's URL in Admin > Nodes must change to port %d, "+ + "such as http://host.docker.internal:%d for a server in Docker Desktop on this Mac.\n", oldPort, settings.port, settings.port, settings.port) + } else { + fmt.Fprintf(&b, "In Admin > Nodes, the node's URL must be one the server can reach. "+ + "For a server in Docker Desktop on this Mac, use http://host.docker.internal:%d.\n", settings.port) + } + _, err := io.WriteString(n.out, b.String()) + return err +} + +// nodeInstallSwap is one installed file: its staged replacement and the +// previous file, kept until the new node is ready. +type nodeInstallSwap struct { + path string + hadOld bool // the previous file is at old() + placed bool // the staged file is at path +} + +func (s nodeInstallSwap) staged() string { return s.path + ".new" } +func (s nodeInstallSwap) old() string { return s.path + ".old" } + +// stage writes the new files beside the installed ones. The binary is copied +// only when this is not already the installed binary. +func (n nodeInstaller) stage(env []byte, key string) ([]nodeInstallSwap, error) { + var swaps []nodeInstallSwap + exe, err := os.Executable() + if err != nil { + return nil, err + } + if !sameFile(exe, n.paths.binary) { + swaps = append(swaps, nodeInstallSwap{path: n.paths.binary}) + if err := copyFile(exe, swaps[0].staged(), 0o755); err != nil { + return swaps, err + } + } + for _, file := range []struct { + path string + data []byte + }{{n.paths.env, env}, {n.paths.key, []byte(key + "\n")}} { + swaps = append(swaps, nodeInstallSwap{path: file.path}) + if err := writeNodeFile(swaps[len(swaps)-1].staged(), file.data, 0o600); err != nil { + return swaps, err + } + } + return swaps, nil +} + +// swapIn moves each previous file aside and its staged file into place, +// recording how far it got for a rollback. +func swapIn(swaps []nodeInstallSwap) error { + for i := range swaps { + s := &swaps[i] + var err error + if s.hadOld, err = s.moveAside(); err != nil { + return err + } + if err := os.Rename(s.staged(), s.path); err != nil { + return err + } + s.placed = true + } + return nil +} + +// moveAside moves the previous file to old() and reports whether old() holds +// one. A file already at old() is the last working one, left by a run that +// stopped mid-swap, so it stays the backup and the unverified file at path is +// dropped. +func (s nodeInstallSwap) moveAside() (hadOld bool, err error) { + if _, err := os.Lstat(s.old()); err == nil { + if err := os.Remove(s.path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return true, err + } + return true, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return false, err + } + err = os.Rename(s.path, s.old()) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return err == nil, err +} + +func discardStaged(swaps []nodeInstallSwap) { + for _, s := range swaps { + _ = os.Remove(s.staged()) + } +} + +// rollback stops the new job, puts the previous files back, and starts the +// previous job again when launchd had it (oldLoaded); without one, it removes +// what the install added. Until the new job is known to be gone it touches no +// file, so a node that may still run keeps its files, and the backups stay. +func (n nodeInstaller) rollback(ctx context.Context, swaps []nodeInstallSwap, oldPlist []byte, oldLoaded bool) error { + if _, err := n.stop(ctx, io.Discard); err != nil { + var backups []string + for _, s := range swaps { + if s.hadOld { + backups = append(backups, s.old()) + } + } + kept := "no previous files were replaced" + if len(backups) > 0 { + kept = "the previous files are kept at " + strings.Join(backups, ", ") + } + return fmt.Errorf("recovery is incomplete: the new node could not be confirmed stopped (%w), so nothing was restored and %s", err, kept) + } + var errs []error + for _, s := range swaps { + if !s.placed { + _ = os.Remove(s.staged()) + } + switch { + case s.hadOld: + errs = append(errs, os.Rename(s.old(), s.path)) + case s.placed: + errs = append(errs, os.Remove(s.path)) + } + } + if oldPlist == nil { + if err := os.Remove(n.paths.plist); err != nil && !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, err) + } + } else { + errs = append(errs, writeNodeFile(n.paths.plist, oldPlist, 0o644)) + } + if err := errors.Join(errs...); err != nil { + return err + } + // The previous install is whole again, so the swap is over. + _ = os.Remove(n.paths.swapPending()) + if oldPlist == nil || !oldLoaded { + return nil + } + return n.bootstrap(ctx) +} + +func uninstallNode(out io.Writer) error { + n, err := newNodeInstaller(out) + if err != nil { + return err + } + ctx, stopSignals := nodeSignalContext() + defer stopSignals() + unlock, err := lockNodeInstall(n.paths.lock) + if err != nil { + return err + } + defer unlock() + // The node takes the default for a drain it cannot read, and so does its + // stop. + if _, extra, err := n.stored(); err == nil { + if drain, err := nodeEnvDrainTimeout(extra); err == nil { + n.drain = drain + } + } + n.drain = max(n.drain, n.installedPlistDrain()) + _, dirErr := os.Stat(n.paths.dir) + _, plistErr := os.Stat(n.paths.plist) + loaded, err := n.loaded(ctx) + if err != nil { + return err + } + if errors.Is(dirErr, fs.ErrNotExist) && errors.Is(plistErr, fs.ErrNotExist) && !loaded { + _, err := fmt.Fprintln(out, "No Silo transcode node is installed.") + return err + } + if _, err := n.stopRunning(ctx, plistErr == nil); err != nil { + return err + } + if err := os.Remove(n.paths.plist); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + if err := os.RemoveAll(n.paths.dir); err != nil { + return err + } + _, err = fmt.Fprintf(out, "Removed the Silo transcode node. Its log stays at %s.\n", n.paths.log) + return err +} + +// waitNodeReady polls the new node's health until it answers 200 as a +// transcode node (node_control_version set) whose key the server took. +// drained reports a node an administrator drained, which stays drained +// across restarts and is ready all the same. +func waitNodeReady(ctx context.Context, settings nodeInstallSettings) (drained bool, err error) { + ctx, cancel := context.WithTimeout(ctx, nodeReadyTimeout) + defer cancel() + client := &http.Client{Timeout: 2 * time.Second} + healthURL := "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(settings.port)) + "/api/v1/health" + ticker := time.NewTicker(nodePollInterval) + defer ticker.Stop() + var last string + for { + health, err := fetchNodeHealth(ctx, client, healthURL) + switch { + case err != nil: + last = err.Error() + case health.NodeControlVersion < 1: + last = "something other than a Silo transcode node answers on the port" + case health.KeyRejected: + return false, fmt.Errorf("the server at %s refused this node's key. Check that it is the key Admin > Nodes on that server last issued "+ + "for this node, and that this Mac's clock is within 30 seconds of the server's", settings.server) + default: + return health.Status == transcodenode.HealthStatusDraining || health.Status == transcodenode.HealthStatusDrained, nil + } + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + return false, ctx.Err() + } + return false, fmt.Errorf("the node was not ready within %d seconds (%s). It opens its port only once the server accepts its key, "+ + "so check the key, that this Mac reaches the server at %s, and that this Mac's clock is right", + int(nodeReadyTimeout/time.Second), last, settings.server) + case <-ticker.C: + } + } +} + +func fetchNodeHealth(ctx context.Context, client *http.Client, healthURL string) (transcodenode.HealthResponse, error) { + var health transcodenode.HealthResponse + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) + if err != nil { + return health, err + } + resp, err := client.Do(req) + if err != nil { + return health, errors.New("no answer on the node's port") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return health, fmt.Errorf("health answered %d", resp.StatusCode) + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&health); err != nil { + return health, errors.New("health answer unreadable") + } + return health, nil +} + +// nodePortTaken reports whether something answers on port at this Mac's +// loopback, which a port Docker Desktop publishes does. It connects rather +// than binds, since a bind on every interface can raise the firewall's prompt. +func nodePortTaken(ctx context.Context, port int) bool { + dialer := net.Dialer{Timeout: time.Second} + for _, host := range []string{"127.0.0.1", "::1"} { + if conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port))); err == nil { + _ = conn.Close() + return true + } + } + return false +} + +// promptNodeKey asks for the key, without echoing it, when stdin is a +// terminal; otherwise it reads the key from stdin's first line. An +// interrupt at the prompt restores the terminal before the process ends. +func promptNodeKey(in io.Reader, out io.Writer) (string, error) { + file, ok := in.(*os.File) + if !ok || !term.IsTerminal(int(file.Fd())) { + if _, err := fmt.Fprintln(out, "Reading the node key from standard input."); err != nil { + return "", err + } + return readNodeKeyLine(in) + } + fd := int(file.Fd()) + state, err := term.GetState(fd) + if err != nil { + return "", err + } + done := make(chan struct{}) + defer close(done) + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(signals) + go func() { + select { + case <-signals: + _ = term.Restore(fd, state) + _, _ = fmt.Fprintln(out) + os.Exit(130) + case <-done: + } + }() + if _, err := fmt.Fprint(out, "Node key from Admin > Nodes (not shown): "); err != nil { + return "", err + } + key, err := term.ReadPassword(fd) + _, _ = fmt.Fprintln(out) + return strings.TrimSpace(string(key)), err +} + +// readNodeKeyLine reads a piped key: the first line of in, trimmed. A key is +// short, so it reads at most 4 KiB. +func readNodeKeyLine(in io.Reader) (string, error) { + line, err := bufio.NewReader(io.LimitReader(in, 4<<10)).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", err + } + return strings.TrimSpace(line), nil +} + +// lockNodeInstall holds an exclusive lock on path until the returned func +// runs, so two installs cannot overlap. The lock file is never removed, so +// every run locks the same file. +func lockNodeInstall(path string) (func(), error) { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) //nolint:gosec // G304: the installer's own lock file. + if err != nil { + return nil, err + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, errors.New("another silo node install or uninstall is running") + } + return nil, err + } + return func() { _ = file.Close() }, nil +} + +// writeNodeFile writes data to path with mode, whatever mode an existing file +// had. +func writeNodeFile(path string, data []byte, mode os.FileMode) error { + return writeNodeFileFrom(path, bytes.NewReader(data), mode) +} + +func writeNodeFileFrom(path string, src io.Reader, mode os.FileMode) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) //nolint:gosec // G304: a path under the user's own Library. + if err != nil { + return err + } + _, err = io.Copy(file, src) + err = errors.Join(err, file.Chmod(mode), file.Sync(), file.Close()) + if err != nil { + _ = os.Remove(path) + } + return err +} + +func copyFile(from, to string, mode os.FileMode) error { + src, err := os.Open(from) //nolint:gosec // G304: this process's own executable. + if err != nil { + return err + } + defer func() { _ = src.Close() }() + return writeNodeFileFrom(to, src, mode) +} + +// sameFile reports whether a and b name one existing file. +func sameFile(a, b string) bool { + aInfo, err := os.Stat(a) + if err != nil { + return false + } + bInfo, err := os.Stat(b) + return err == nil && os.SameFile(aInfo, bInfo) +} + +func fileSize(path string) int64 { + if info, err := os.Stat(path); err == nil { + return info.Size() + } + return 0 +} + +// nodeLogTail returns the last lines the log gained after offset from, or the +// log's last lines when it gained none. +func nodeLogTail(path string, from int64) string { + const maxLines, maxBytes = 20, 64 << 10 + file, err := os.Open(path) //nolint:gosec // G304: the node's own log. + if err != nil { + return " (no log)" + } + defer func() { _ = file.Close() }() + size := fileSize(path) + if from >= size { + from = 0 + } + from = max(from, size-maxBytes) + data, err := io.ReadAll(io.NewSectionReader(file, from, size-from)) + if err != nil { + return " (log unreadable)" + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + if len(lines) == 1 && lines[0] == "" { + return " (empty)" + } + return " " + strings.Join(lines, "\n ") +} diff --git a/internal/nodeinstall/node_install_darwin_test.go b/internal/nodeinstall/node_install_darwin_test.go new file mode 100644 index 0000000000..8211e65685 --- /dev/null +++ b/internal/nodeinstall/node_install_darwin_test.go @@ -0,0 +1,460 @@ +//go:build darwin + +package nodeinstall + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// fakeLaunchctl points launchctlPath at a script that logs its calls to +// /calls and answers bootout with bootoutStatus. print reports the job +// loaded until /gone exists, which the first print after a bootout +// creates, so the job is still shutting down at that print and gone at the +// next. +func fakeLaunchctl(t *testing.T, bootoutStatus string) (dir string) { + t.Helper() + dir = t.TempDir() + script := `#!/bin/sh +dir='` + dir + `' +echo "$*" >> "$dir/calls" +case "$1" in +print) [ -e "$dir/gone" ] && exit 113; [ -e "$dir/booted" ] && touch "$dir/gone"; exit 0 ;; +bootout) touch "$dir/booted"; exit ` + bootoutStatus + ` ;; +esac +exit 0 +` + path := filepath.Join(dir, "launchctl") + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + previous := launchctlPath + launchctlPath = path + t.Cleanup(func() { launchctlPath = previous }) + return dir +} + +// fakeDrainingLaunchctl points launchctlPath at a script whose job runs as pid +// 100 until a kill. The first print after the kill still shows pid 100; the +// next shows afterPID (none when empty), as the job an older plist keeps alive +// would. A bootout then removes the job by the second print after it. +func fakeDrainingLaunchctl(t *testing.T, afterPID string) (dir string) { + t.Helper() + dir = t.TempDir() + script := `#!/bin/sh +dir='` + dir + `' +echo "$*" >> "$dir/calls" +pidline() { [ -n "$1" ] && printf 'state = running\n\tpid = %s\n\tpid-local endpoints = {\n\t\tpid = 1\n\t}\n' "$1"; return 0; } +case "$1" in +print) + [ -e "$dir/gone" ] && exit 113 + [ -e "$dir/booted" ] && { touch "$dir/gone"; exit 0; } + if [ -e "$dir/killed" ]; then + if [ -e "$dir/exited" ]; then pidline '` + afterPID + `'; else touch "$dir/exited"; pidline 100; fi + else + pidline 100 + fi + exit 0 ;; +kill) touch "$dir/killed" ;; +bootout) touch "$dir/booted" ;; +esac +exit 0 +` + path := filepath.Join(dir, "launchctl") + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + previous := launchctlPath + launchctlPath = path + t.Cleanup(func() { launchctlPath = previous }) + return dir +} + +func fakeInstaller(t *testing.T) nodeInstaller { + return nodeInstaller{paths: nodeInstallPathsIn(t.TempDir()), service: "gui/0/" + nodeLaunchdLabel, domain: "gui/0", + out: io.Discard, drain: transcodenode.DefaultDrainTimeout} +} + +func launchctlCalls(t *testing.T, dir string) string { + t.Helper() + calls, err := os.ReadFile(filepath.Join(dir, "calls")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + return string(calls) +} + +// A bootout that answers EINPROGRESS went out: stop waits for the job, still +// loaded at the next print, to go. +func TestStopTreatsBootoutInProgressAsSent(t *testing.T) { + dir := fakeLaunchctl(t, "36") + sent, err := fakeInstaller(t).stop(context.Background(), io.Discard) + if !sent || err != nil { + t.Fatalf("stop = %v, %v; want sent and no error", sent, err) + } + if calls := launchctlCalls(t, dir); !strings.Contains(calls, "bootout") || strings.Count(calls, "print") < 3 { + t.Fatalf("launchctl calls:\n%s\nwant a bootout, then prints until the job is gone", calls) + } +} + +// stop sends the running node SIGTERM and waits for that process to exit +// before the bootout, since launchd kills a job a minute after its bootout +// whatever ExitTimeOut asks. A job an older plist restarts under a new pid +// still goes on to the bootout. +func TestStopDrainsBeforeTheBootout(t *testing.T) { + for _, afterPID := range []string{"", "200"} { + dir := fakeDrainingLaunchctl(t, afterPID) + sent, err := fakeInstaller(t).stop(context.Background(), io.Discard) + if !sent || err != nil { + t.Fatalf("after pid %q: stop = %v, %v; want sent and no error", afterPID, sent, err) + } + calls := strings.Split(strings.TrimSpace(launchctlCalls(t, dir)), "\n") + want := []string{"print", "kill SIGTERM", "print", "print", "bootout", "print", "print"} + if len(calls) != len(want) { + t.Fatalf("after pid %q: launchctl calls %q; want %q", afterPID, calls, want) + } + for i := range want { + if !strings.HasPrefix(calls[i], want[i]) { + t.Fatalf("after pid %q: launchctl calls %q; want %q", afterPID, calls, want) + } + } + } +} + +// A node still draining at the stop's deadline is booted out all the same, +// and the installer says so. +func TestStopBootsOutANodeStillDrainingAtTheDeadline(t *testing.T) { + previous := nodeStopWait + nodeStopWait = func(time.Duration) time.Duration { return time.Second } + t.Cleanup(func() { nodeStopWait = previous }) + dir := fakeDrainingLaunchctl(t, "100") + var out bytes.Buffer + sent, err := fakeInstaller(t).stop(context.Background(), &out) + if !sent || err != nil { + t.Fatalf("stop = %v, %v; want sent and no error", sent, err) + } + if !strings.Contains(launchctlCalls(t, dir), "bootout") { + t.Fatal("no bootout after the deadline") + } + if !strings.Contains(out.String(), "still finishing streams") { + t.Fatalf("output %q does not say the node is stopped mid-stream", out.String()) + } +} + +// A rollback that cannot confirm the new job stopped restores nothing and +// keeps the backups, which a node that may still run must not lose. +func TestRollbackKeepsBackupsWhenTheStopFails(t *testing.T) { + dir := t.TempDir() + script := "#!/bin/sh\necho \"$*\" >> '" + dir + "/calls'\nexit 1\n" + if err := os.WriteFile(filepath.Join(dir, "launchctl"), []byte(script), 0o700); err != nil { + t.Fatal(err) + } + previous := launchctlPath + launchctlPath = filepath.Join(dir, "launchctl") + t.Cleanup(func() { launchctlPath = previous }) + n := fakeInstaller(t) + if err := os.MkdirAll(filepath.Dir(n.paths.plist), 0o700); err != nil { + t.Fatal(err) + } + files := map[string]string{n.paths.plist: "new plist", n.paths.env: "new env", n.paths.env + ".old": "old env"} + for path, data := range files { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + } + err := n.rollback(context.Background(), []nodeInstallSwap{{path: n.paths.env, hadOld: true, placed: true}}, []byte("old plist"), true) + if err == nil || !strings.Contains(err.Error(), "recovery is incomplete") || !strings.Contains(err.Error(), n.paths.env+".old") { + t.Fatalf("rollback = %v; want incomplete recovery naming the backup", err) + } + for path, want := range files { + if got, err := os.ReadFile(path); err != nil || string(got) != want { + t.Errorf("%s = %q, %v; want %q left in place", path, got, err, want) + } + } + if strings.Contains(launchctlCalls(t, dir), "bootstrap") { + t.Fatal("bootstrapped the previous job while the new one may still run") + } +} + +// A run stopped mid-swap leaves node.env and node-key only at their backups; +// a rerun without flags takes its settings from them. Without either, the +// install is new and needs --server. +func TestStoredReadsAnInterruptedSwapsBackups(t *testing.T) { + n := fakeInstaller(t) + if err := os.MkdirAll(n.paths.dir, 0o700); err != nil { + t.Fatal(err) + } + if stored, _, err := n.stored(); err != nil || stored != (nodeInstallSettings{}) { + t.Fatalf("no install: stored = %+v, %v; want nothing", stored, err) + } + if _, err := resolveNodeInstall(nodeInstallSettings{}, nodeInstallSettings{}); err == nil || !strings.Contains(err.Error(), "--server is required") { + t.Fatalf("no install: resolve = %v; want --server required", err) + } + env := "MODE=\"transcode\"\nSILO_CORE_URL=\"http://localhost:8090\"\nPORT=\"8200\"\nSILO_NODE_DRAIN_TIMEOUT=\"15m\"\n" + for path, data := range map[string]string{n.paths.env + ".old": env, n.paths.key + ".old": "old-key\n"} { + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + } + stored, extra, err := n.stored() + want := nodeInstallSettings{server: "http://localhost:8090", key: "old-key", port: 8200} + if err != nil || stored != want || extra[transcodenode.EnvDrainTimeout] != "15m" { + t.Fatalf("stored = %+v, %v, %v; want %+v and the drain kept", stored, extra, err, want) + } +} + +// --server is a plain URL: the probe and the node join paths onto it, and +// userinfo would go out as Basic auth and show in messages. +func TestValidateRefusesServerURLsWithMore(t *testing.T) { + for _, server := range []string{"http://user:pass@localhost:8090", "http://localhost:8090/?a=b", "http://localhost:8090?", "http://localhost:8090/#x", "http://localhost:8090#"} { + err := nodeInstallSettings{server: server, port: 8100}.validateAllButKey() + if err == nil || !strings.Contains(err.Error(), "plain http or https URL") { + t.Errorf("--server %q: %v; want refused", server, err) + } + } +} + +// The readiness probe keeps the path of a server behind a prefix, with or +// without a trailing slash. +func TestCheckServerReadyKeepsAPathPrefix(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/silo/api/v1/ready" { + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + for _, url := range []string{server.URL + "/silo", server.URL + "/silo/"} { + if err := (nodeInstallSettings{server: url, port: 8100}).validateAllButKey(); err != nil { + t.Fatalf("--server %q: %v", url, err) + } + if err := checkServerReady(context.Background(), url); err != nil { + t.Fatalf("--server %q: %v", url, err) + } + } +} + +// A rollback restores the previous plist, and starts its job only when +// launchd had it before the install. +func TestRollbackRestoresWhetherTheJobWasLoaded(t *testing.T) { + for _, oldLoaded := range []bool{false, true} { + dir := fakeLaunchctl(t, "0") + if err := os.WriteFile(filepath.Join(dir, "gone"), nil, 0o600); err != nil { + t.Fatal(err) + } + n := fakeInstaller(t) + if err := os.MkdirAll(filepath.Dir(n.paths.plist), 0o700); err != nil { + t.Fatal(err) + } + if err := n.rollback(context.Background(), nil, []byte("old plist"), oldLoaded); err != nil { + t.Fatal(err) + } + if plist, err := os.ReadFile(n.paths.plist); err != nil || string(plist) != "old plist" { + t.Fatalf("plist = %q, %v; want the previous one", plist, err) + } + if bootstrapped := strings.Contains(launchctlCalls(t, dir), "bootstrap"); bootstrapped != oldLoaded { + t.Fatalf("old job loaded %v: bootstrapped %v", oldLoaded, bootstrapped) + } + } +} + +// A retry after a run that stopped mid-swap keeps the backup that run made, +// not the unverified file it left in place. +func TestSwapInKeepsAnExistingBackup(t *testing.T) { + dir := t.TempDir() + write := func(name, data string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o600); err != nil { + t.Fatal(err) + } + } + write("f", "unverified") + write("f.old", "last working") + write("f.new", "staged") + write("g", "current") + write("g.new", "staged") + swaps := []nodeInstallSwap{{path: filepath.Join(dir, "f")}, {path: filepath.Join(dir, "g")}} + if err := swapIn(swaps); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"f": "staged", "f.old": "last working", "g": "staged", "g.old": "current"} { + if got, err := os.ReadFile(filepath.Join(dir, name)); err != nil || string(got) != want { + t.Errorf("%s = %q, %v; want %q", name, got, err, want) + } + } + if !swaps[0].hadOld || !swaps[1].hadOld { + t.Fatalf("swaps = %+v, want both with a backup", swaps) + } +} + +// The install lock sits outside the folder uninstall removes, so removing the +// folder under a held lock does not let another run take a new one. +func TestNodeInstallLockOutlivesTheInstallFolder(t *testing.T) { + paths := nodeInstallPathsIn(t.TempDir()) + if strings.HasPrefix(paths.lock, paths.dir+string(filepath.Separator)) { + t.Fatalf("lock %s is inside %s", paths.lock, paths.dir) + } + unlock, err := lockNodeInstall(paths.lock) + if err != nil { + t.Fatal(err) + } + defer unlock() + for path, want := range map[string]os.FileMode{filepath.Dir(paths.lock): os.ModeDir | 0o700, paths.lock: 0o600} { + if info, err := os.Stat(path); err != nil || info.Mode() != want { + t.Fatalf("%s: %v, %v; want mode %v", path, info, err, want) + } + } + if err := os.RemoveAll(paths.dir); err != nil { + t.Fatal(err) + } + if again, err := lockNodeInstall(paths.lock); err == nil { + again() + t.Fatal("a second run took the lock while the first held it") + } +} + +// --new-key asks for the key even when the install has one; without it, a +// stored key is kept without asking. +func TestAskNodeKey(t *testing.T) { + stored := nodeInstallSettings{server: "http://localhost:8090", key: "stored", port: 8100} + prompted := func(key string) func() (string, error) { return func() (string, error) { return key, nil } } + noPrompt := func() (string, error) { t.Fatal("asked for a key"); return "", nil } + + if key, err := askNodeKey(stored, nodeInstallSettings{}, noPrompt); err != nil || key != "" { + t.Fatalf("rerun: key %q, %v; want the stored key kept", key, err) + } + if key, err := askNodeKey(stored, nodeInstallSettings{key: nodeKeyPrompt}, prompted("new")); err != nil || key != "new" { + t.Fatalf("--new-key: key %q, %v; want the entered key", key, err) + } + if _, err := askNodeKey(stored, nodeInstallSettings{key: nodeKeyPrompt}, prompted("")); err == nil { + t.Fatal("--new-key with no key entered kept the stored key") + } +} + +// A key piped in on standard input is its first line, trimmed; the key never +// has to be a command-line argument. +func TestPromptNodeKeyReadsAPipedKey(t *testing.T) { + for in, want := range map[string]string{"snk1_1_abc\n": "snk1_1_abc", " snk1_1_abc ": "snk1_1_abc", "snk1_1_abc\nmore\n": "snk1_1_abc", "": ""} { + if key, err := promptNodeKey(strings.NewReader(in), io.Discard); err != nil || key != want { + t.Fatalf("piped %q: key %q, %v; want %q", in, key, err, want) + } + } +} + +// Without the swap marker, .old files are a finished run's leftovers and go; +// with it, they are the last working install and stay. +func TestDropStaleBackupsFollowsTheSwapMarker(t *testing.T) { + n := nodeInstaller{paths: nodeInstallPathsIn(t.TempDir())} + if err := os.MkdirAll(n.paths.dir, 0o700); err != nil { + t.Fatal(err) + } + writeBackups := func() { + for _, path := range n.paths.installedFiles() { + if err := os.WriteFile(path+".old", []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + } + } + countBackups := func() (n2 int) { + for _, path := range n.paths.installedFiles() { + if _, err := os.Stat(path + ".old"); err == nil { + n2++ + } + } + return n2 + } + + writeBackups() + if err := os.WriteFile(n.paths.swapPending(), nil, 0o600); err != nil { + t.Fatal(err) + } + n.dropStaleBackups() + if got := countBackups(); got != 3 { + t.Fatalf("with the marker, %d of 3 backups kept; want all", got) + } + + if err := os.Remove(n.paths.swapPending()); err != nil { + t.Fatal(err) + } + n.dropStaleBackups() + if got := countBackups(); got != 0 { + t.Fatalf("without the marker, %d backups left; want none", got) + } +} + +// A launchctl print error while the node drains doesn't end the wait: the +// bootout comes only once the signaled process is gone. +func TestStopKeepsDrainingThroughAPrintError(t *testing.T) { + dir := t.TempDir() + script := `#!/bin/sh +dir='` + dir + `' +echo "$*" >> "$dir/calls" +case "$1" in +print) + [ -e "$dir/gone" ] && exit 113 + [ -e "$dir/booted" ] && { touch "$dir/gone"; exit 0; } + if [ -e "$dir/killed" ]; then + n=$(cat "$dir/polls" 2>/dev/null || echo 0); n=$((n+1)); echo $n > "$dir/polls" + case $n in + 1) echo "Could not find service" >&2; exit 5 ;; + 2) printf 'state = running\n\tpid = 100\n' ;; + *) printf 'state = not running\n' ;; + esac + else + printf 'state = running\n\tpid = 100\n' + fi + exit 0 ;; +kill) touch "$dir/killed" ;; +bootout) touch "$dir/booted" ;; +esac +exit 0 +` + path := filepath.Join(dir, "launchctl") + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + previous := launchctlPath + launchctlPath = path + t.Cleanup(func() { launchctlPath = previous }) + + if sent, err := fakeInstaller(t).stop(context.Background(), io.Discard); !sent || err != nil { + t.Fatalf("stop = %v, %v; want sent and no error", sent, err) + } + calls := strings.Split(strings.TrimSpace(launchctlCalls(t, dir)), "\n") + want := []string{"print", "kill SIGTERM", "print", "print", "print", "bootout"} + for i := range want { + if i >= len(calls) || !strings.HasPrefix(calls[i], want[i]) { + t.Fatalf("launchctl calls %q; want them to start %q", calls, want) + } + } +} + +// The installed plist records the drain the running node started with, which +// bounds its stop even when node.env now asks for less. +func TestInstalledPlistDrain(t *testing.T) { + n := fakeInstaller(t) + if got := n.installedPlistDrain(); got != 0 { + t.Fatalf("no plist: drain %v; want 0", got) + } + if err := os.MkdirAll(filepath.Dir(n.paths.plist), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(n.paths.plist, renderNodePlist(n.paths, 20*time.Minute), 0o600); err != nil { + t.Fatal(err) + } + if got := n.installedPlistDrain(); got != 20*time.Minute { + t.Fatalf("drain %v; want 20m from the plist's ExitTimeOut", got) + } +} diff --git a/internal/nodeinstall/node_install_other.go b/internal/nodeinstall/node_install_other.go new file mode 100644 index 0000000000..f6cfbdab37 --- /dev/null +++ b/internal/nodeinstall/node_install_other.go @@ -0,0 +1,14 @@ +//go:build !darwin + +package nodeinstall + +import ( + "errors" + "io" +) + +var errNodeInstallMacOnly = errors.New("the node installer is for macOS. On Linux, run the Silo image with MODE=transcode as a transcode node") + +func installNode([]string, io.Reader, io.Writer) error { return errNodeInstallMacOnly } + +func uninstallNode(io.Writer) error { return errNodeInstallMacOnly } diff --git a/internal/playback/remux.go b/internal/playback/remux.go index 5115b498d2..9119175768 100644 --- a/internal/playback/remux.go +++ b/internal/playback/remux.go @@ -99,6 +99,12 @@ func discoverFFmpegPath(goos string, lookPath func(string) (string, error)) stri return ffmpegExecutable } +// HomebrewFFmpegFull reports whether path is Homebrew's keg-only ffmpeg-full, +// the build discovery prefers on a Mac. +func HomebrewFFmpegFull(path string) bool { + return path == homebrewFFmpegFullAppleSilicon || path == homebrewFFmpegFullIntel +} + // ResolveFFmpegPath returns the ffmpeg binary the playback pipeline executes // for the given configured path: the configured path when set, otherwise the // process-global discovery (jellyfin-ffmpeg install, then PATH). Capability diff --git a/internal/transcodenode/drain.go b/internal/transcodenode/drain.go index b8dc15b124..278bea1d15 100644 --- a/internal/transcodenode/drain.go +++ b/internal/transcodenode/drain.go @@ -17,6 +17,11 @@ const ( HealthStatusDrained = "drained" ) +// DefaultDrainTimeout is how long a transcode node that received SIGTERM +// drains by default. It lets most episodes and a long tail of movie sessions +// finish, while keeping a rolling upgrade of several nodes bounded. +const DefaultDrainTimeout = 10 * time.Minute + // drainPollInterval is how often WaitDrained and announceWhenDrained look at // the node's work. const drainPollInterval = 250 * time.Millisecond diff --git a/internal/transcodenode/env.go b/internal/transcodenode/env.go new file mode 100644 index 0000000000..3a11e6e536 --- /dev/null +++ b/internal/transcodenode/env.go @@ -0,0 +1,53 @@ +package transcodenode + +import ( + "errors" + "math" + "net/url" + "strconv" + "strings" + "time" +) + +// Mode is the MODE that runs a process as a transcode node. +const Mode = "transcode" + +// EnvCoreURL is the URL a transcode node reaches the core at. With its own +// key (EnvNodeKey) it is all the node needs: everything else comes from the +// core. +const EnvCoreURL = "SILO_CORE_URL" + +// EnvNodeKey is the transcode node's own key, issued by an administrator. The +// node verifies every request the core signs to it with this key. +const EnvNodeKey = "SILO_NODE_KEY" + +// ParseCoreURL parses raw as SILO_CORE_URL must be: an absolute http or https +// URL. +func ParseCoreURL(raw string) (*url.URL, bool) { + parsed, err := url.Parse(raw) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return nil, false + } + return parsed, true +} + +// EnvDrainTimeout bounds how long a transcode node that received SIGTERM keeps +// serving its sessions before it shuts down. 0 turns the drain off. +const EnvDrainTimeout = "SILO_NODE_DRAIN_TIMEOUT" + +// ParseDrainTimeout parses SILO_NODE_DRAIN_TIMEOUT: a Go duration ("15m") or +// whole seconds ("900"). Empty means DefaultDrainTimeout. Anything else, such +// as a negative value, is an error. +func ParseDrainTimeout(value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return DefaultDrainTimeout, nil + } + if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 && seconds <= math.MaxInt64/int(time.Second) { + return time.Duration(seconds) * time.Second, nil + } + if timeout, err := time.ParseDuration(value); err == nil && timeout >= 0 { + return timeout, nil + } + return 0, errors.New(EnvDrainTimeout + " must be a duration such as 15m or whole seconds such as 900") +} diff --git a/web/embed.go b/web/embed.go index 5f8929316e..6e1548bc03 100644 --- a/web/embed.go +++ b/web/embed.go @@ -1,3 +1,5 @@ +//go:build !silo_node + package web import "embed" diff --git a/web/embed_node.go b/web/embed_node.go new file mode 100644 index 0000000000..2867e52f1f --- /dev/null +++ b/web/embed_node.go @@ -0,0 +1,9 @@ +//go:build silo_node + +package web + +import "embed" + +// DistFS is empty in the transcode node build (tag silo_node), which serves +// no web app. +var DistFS embed.FS