diff --git a/CHANGELOG.md b/CHANGELOG.md index d92d9a3..a62b1bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,36 @@ one waits for a major. context` asks the registry and falls back to the rootfs tarball, which is the path the published 25.12.5 images were already built through. +### Fixed + +- **One router's `extra_packages` no longer cancels every other router.** + `docker compose build` puts the whole lab in one buildkit solve, and buildkit + cancels the solve on the first target that fails — so the extras step, which + ran under `set -eu` with no guard, turned one unusable file into a failed + lab. Measured 2026-09-04 on a three-router stand: one `.ipk` with an + unsatisfiable dependency exited 255 and took `#16 CANCELED`, `#13 CANCELED` + and `owlab: build failed: exit status 1` with it. Both release lines did it — + it is `set -eu`, not the package manager: the same stand with a `.apk` that + apk answers `unable to select packages` exited 27 and cancelled its + neighbour. There is nothing to configure around it, either: neither `docker + compose build` nor `docker buildx bake` has a `--keep-going`. + + The extras step now installs the set as before, and only if that fails + installs each file on its own, in the staged order, so the good ones still + land. The successful path is byte-for-byte the run it was. + +- **A package that did not install is now said out loud instead of being + fatal.** Tolerating a failure silently would be worse than the cancelled + builds it replaces, so the retry names each file it could not install, writes + the list to `/etc/owlab/extras-failed`, and `owlab up` reads it back off each + running router and prints it under the ready table: + `! owrt2410 is running WITHOUT luci-app-example_1.0_all.ipk`. `owlab up` + exits non-zero when it prints that — the lab is up, and it is not what the + config describes — and `owlab test` gains an `extra_packages` step that fails + the router before any assertion runs against a box missing what it was told + to have. Read as-built at any time with + `owlab exec -- cat /etc/owlab/extras-failed`. + ## [0.5.6] - 2026-09-04 ### Fixed diff --git a/cmd/owlab/commands.go b/cmd/owlab/commands.go index f3e2df6..2d3ccee 100644 --- a/cmd/owlab/commands.go +++ b/cmd/owlab/commands.go @@ -1,8 +1,10 @@ package main import ( + "bytes" "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -94,13 +96,90 @@ func (a *app) up(ctx context.Context, args []string) error { } } - if *noWait { - return a.printReady(routers, nil) + var ready map[string]bool + if !*noWait { + ready = a.waitForLuCI(ctx, routers) } - ready := a.waitForLuCI(ctx, routers) - return a.printReady(routers, ready) + if err := a.printReady(routers, ready); err != nil { + return err + } + // After the table, because it is what should be left on screen — and + // after the build, because the build no longer stops for it. + return a.reportMissingExtras(ctx, containers) +} + +// extrasFailedPath is where an image build records the extra_packages it could +// not install. See the extras layer in images/Dockerfile. +// +// Read back off the router rather than scraped out of the build log, for two +// reasons: the log is thousands of lines of buildkit progress that nobody +// reads, and a router started from a cached image never printed one at all. +const extrasFailedPath = "/etc/owlab/extras-failed" + +// missingExtras asks one router which of its extra_packages did not install. +// +// Empty for almost every router: the file exists only when a build had +// something to record. +func missingExtras(ctx context.Context, run syncpkg.Exec) []string { + var buf bytes.Buffer + // A missing file is the normal case, not an error to propagate — and any + // other failure here (a router that stopped, an engine that went away) is + // already reported by whatever else is talking to the same router. + if err := run(ctx, "cat "+extrasFailedPath+" 2>/dev/null || true", nil, &buf); err != nil { + return nil + } + var out []string + for _, line := range strings.Split(buf.String(), "\n") { + if line = strings.TrimSpace(line); line != "" { + out = append(out, line) + } + } + return out +} + +// reportMissingExtras names, per router, the extra_packages its image was +// built without. +// +// This is the half of issue #12 that is not "stop cancelling the other +// routers". One bad file no longer fails the build, and tolerance with no +// report is silence — which is worse than the failure it replaced: the +// developer named a file, the router does not have it, and nothing but a line +// in the middle of a buildkit log would say so. Hence the non-zero exit as +// well: a script that runs `owlab up` before its own checks must not read +// "lab is up" as "lab is what the config describes". +func (a *app) reportMissingExtras(ctx context.Context, routers []*config.Router) error { + var bad []string + for _, r := range routers { + if len(r.Extra) == 0 { + continue + } + run, err := a.execFor(r) + if err != nil { + continue + } + missing := missingExtras(ctx, run) + if len(missing) == 0 { + continue + } + bad = append(bad, r.ID) + fmt.Fprintf(os.Stderr, "\n! %s is running WITHOUT %s\n", r.ID, strings.Join(missing, ", ")) + } + if len(bad) == 0 { + return nil + } + fmt.Fprintf(os.Stderr, + "! Every other router built and started; only these packages are missing.\n"+ + "! The package manager said why during the build — one router at a time shows it again:\n"+ + "! owlab up --rebuild %s\n", bad[0]) + return errExtrasMissing } +// errExtrasMissing is the exit status of an `owlab up` that started every +// router and could not put a named package on one of them. A plain sentinel: +// the routers and packages have already been printed one line each, and an +// "owlab: extra_packages missing" summary on top would bury which is which. +var errExtrasMissing = errors.New("extra_packages missing") + // resolveBaseImages asks the registry whether each router's upstream image // exists, and switches the ones that do not onto the rootfs tarball. // diff --git a/cmd/owlab/extras_test.go b/cmd/owlab/extras_test.go new file mode 100644 index 0000000..9546a52 --- /dev/null +++ b/cmd/owlab/extras_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "errors" + "io" + "strings" + "testing" +) + +// The record a build leaves behind is the only thing standing between "one bad +// package no longer fails the build" and "one bad package is never mentioned +// again" (issue #12). Reading it wrong is silent in exactly the direction that +// matters: an empty list means "the router has everything it was told to have". +func TestMissingExtrasReadsOneNamePerLine(t *testing.T) { + cases := []struct { + name string + out string + want []string + }{ + // The overwhelming majority: `cat` of a file that is not there, which + // the script turns into empty output rather than a failure. + {"nothing recorded", "", nil}, + {"one package", "01-example_1.0_all.ipk\n", []string{"01-example_1.0_all.ipk"}}, + { + "several, with the blank line a trailing newline leaves", + "example-daemon_1.0_all.ipk\nluci-app-example_1.0_all.ipk\n\n", + []string{"example-daemon_1.0_all.ipk", "luci-app-example_1.0_all.ipk"}, + }, + // docker exec hands back \r\n from a container with a tty attached. + {"carriage returns", "example_1.0_all.ipk\r\n", []string{"example_1.0_all.ipk"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + run := func(_ context.Context, _ string, _ []byte, out io.Writer) error { + _, err := io.WriteString(out, tc.out) + return err + } + got := missingExtras(t.Context(), run) + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Errorf("missingExtras = %q, want %q", got, tc.want) + } + }) + } +} + +// A router that cannot be reached is not a router that is missing a package. +// Reporting one as the other would put "is running WITHOUT" against a box that +// was simply stopped, and send the developer looking at the wrong thing. +func TestMissingExtrasReportsNothingWhenTheRouterCannotAnswer(t *testing.T) { + run := func(_ context.Context, _ string, _ []byte, out io.Writer) error { + _, _ = io.WriteString(out, "Error: No such container\n") + return errors.New("exit status 1") + } + if got := missingExtras(t.Context(), run); got != nil { + t.Errorf("missingExtras = %q, want none", got) + } +} diff --git a/cmd/owlab/main.go b/cmd/owlab/main.go index 71e4824..7f4e1b9 100644 --- a/cmd/owlab/main.go +++ b/cmd/owlab/main.go @@ -154,10 +154,11 @@ func main() { if errors.Is(err, context.Canceled) { os.Exit(130) } - // A failed assertion has already been printed, one line per failure. - // Repeating it as "owlab: test failed" would bury the reason under a - // summary of it. - if errors.Is(err, errTestFailed) { + // A failed assertion, and a router started without a package its + // config named, have both already been printed one line each. + // Repeating either as "owlab: test failed" would bury the reason + // under a summary of it. + if errors.Is(err, errTestFailed) || errors.Is(err, errExtrasMissing) { os.Exit(1) } fmt.Fprintln(os.Stderr, "owlab: "+err.Error()) diff --git a/cmd/owlab/test.go b/cmd/owlab/test.go index c7d88b6..f280194 100644 --- a/cmd/owlab/test.go +++ b/cmd/owlab/test.go @@ -200,6 +200,15 @@ func (a *app) test(ctx context.Context, args []string) error { if err != nil { record(checkpkg.Result{Check: "reach", Kind: "up", Detail: err.Error()}) } else { + // Before anything is installed or asserted. A bad + // extra_packages file no longer fails the image build + // (issue #12), so this step is the only thing left that can + // fail a run whose router is missing a package the config + // named — and asserting against such a router would report a + // pass for something that was never on it. + if len(r.Extra) > 0 && r.Fidelity != config.VM { + record(extrasStep(ctx, run, r)) + } if len(files) > 0 || len(installs) > 0 { record(a.testInstall(ctx, r, run, files, installs, feedSpec)) } @@ -255,6 +264,21 @@ func (a *app) test(ctx context.Context, args []string) error { // errTestFailed reports assertion failures without printing anything more. var errTestFailed = errors.New("test failed") +// extrasStep turns the image's own record of failed extra_packages into a test +// step, so a run cannot pass on a router the build could not finish equipping. +// +// Only for the container tier: a VM installs its extra_packages over ssh +// during provisioning, which fails as itself and never reaches here. +func extrasStep(ctx context.Context, run syncpkg.Exec, r *config.Router) checkpkg.Result { + res := checkpkg.Result{Check: "extra_packages", Kind: checkpkg.KindPackage, OK: true} + if missing := missingExtras(ctx, run); len(missing) > 0 { + res.OK = false + res.Detail = "built without " + strings.Join(missing, ", ") + + " — `owlab up --rebuild " + r.ID + "` shows what the package manager said" + } + return res +} + // testReport is the --json document. The schema field is first because // consumers of a machine-readable output need a way to tell when it changed. type testReport struct { diff --git a/docs/internals.md b/docs/internals.md index 6ca98af..1a1b33e 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -435,7 +435,7 @@ the two package managers do not share a naming scheme: the same release publishes `luci-theme-footstrap-0.11.5-r1.apk` and `luci-theme-footstrap_0.11.5-r1_all.ipk`. -Four things about how these are installed, each of which was a bug first. +Five things about how these are installed, each of which was a bug first. **Downloaded on the host.** A stock OpenWrt rootfs has no `curl` and its busybox `wget` cannot do TLS, so an in-image download would depend on which @@ -449,7 +449,8 @@ packages depend on each other — `luci-app-example` requires `example-daemon`, before it — which apk refuses outright with `unable to select packages`. **All of them in one command.** Handed the whole set, the package manager -resolves among them and the ordering stops mattering at all. +resolves among them and the ordering stops mattering — until the retry below, +which is one file at a time and leans on the staged order again. **`--force-overwrite`.** Their dependencies routinely replace a file the stock image already owns: @@ -464,9 +465,27 @@ exactly what was asked for. ImmortalWrt already ships `dnsmasq-full`, so without this the same config produced a different router on OpenWrt 24.10 than on the other three. -**A failure here is fatal**, unlike a feed name. These are named by URL — the -developer said "install this file" — and a build that reported success without -it would hand them a router quietly missing the thing they are testing against. +**A failure here is neither swallowed nor fatal**, unlike a feed name. These +are named by URL — the developer said "install this file" — so a build that +reported success without one would hand them a router quietly missing the thing +they are testing against. + +Fatal was the first answer and cost more than it saved. `docker compose build` +puts every router in one buildkit solve, and buildkit cancels the solve on the +first target that fails, so one unusable file killed every other router in the +lab. Measured 2026-09-04 on a three-router stand: opkg exits 255 on an +unsatisfiable dependency, apk exits 27 on `unable to select packages`, and both +left `CANCELED` neighbours — it is `set -eu`, not the package manager. Nor is +there a knob: neither `docker compose build` nor `docker buildx bake` has a +`--keep-going`. + +So the set is installed as one, then one file at a time if that failed, and +what is still missing is written to `/etc/owlab/extras-failed` — which +`owlab up` reads back off the running router and reports under its table before +exiting non-zero, and which `owlab test` fails on as an `extra_packages` step. +The record is what makes tolerance honest rather than silent: a build log +scrolls past, and a router started from a cached image never printed one at +all. These are installed **without signature verification**. Projects publishing this way usually sign with usign and ship a `.sig` beside the artifact, but diff --git a/docs/reference.md b/docs/reference.md index 53777e5..729b79f 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -166,10 +166,20 @@ Downloads happen on the host and are cached in `.owlab/cache/`, so `up` does not re-fetch them. They have to: a stock OpenWrt rootfs has no `curl`, and its busybox `wget` cannot do TLS. -A failure here is fatal, unlike a name in `packages:` that a particular feed -happens not to carry. These are named by URL — you said "install this file" — -and a build that reported success without it would hand you a router quietly -missing the thing you are testing against. They are also installed with +A failure here is neither passed over nor fatal, unlike a name in `packages:` +that a particular feed happens not to carry. These are named by URL — you said +"install this file" — so owlab will not report success without one. It will +not fail the build either: every router in a lab is built in one buildkit +solve, and one failing target cancels all the others, so a single package would +take the whole lab down. + +What happens instead: the set is installed together, then one file at a time if +that failed, so the good ones still land. Whatever is still missing is named by +`owlab up` after its table, is failed on by `owlab test`, and is left on the +router in `/etc/owlab/extras-failed`. `owlab up` exits non-zero when it has to +print that — the lab is running, and it is not what your config describes. + +They are also installed with `--force-overwrite`, because their dependencies routinely replace a file the stock image already owns: `luci-app-openclash` pulls `dnsmasq-full`, which ships `/etc/init.d/dnsmasq` and collides with `dnsmasq`. Without it the same diff --git a/docs/reference_ru.md b/docs/reference_ru.md index b8f42f1..ba67f40 100644 --- a/docs/reference_ru.md +++ b/docs/reference_ru.md @@ -163,10 +163,20 @@ defaults: их заново. Иначе и нельзя: в стоковом rootfs нет `curl`, а его busybox `wget` не умеет TLS. -Провал здесь фатален, в отличие от имени из `packages:`, которого конкретный -фид может законно не иметь. Эти названы по URL — вы сказали «поставь этот файл» -— и сборка, отрапортовавшая успех без него, отдала бы вам роутер, тихо лишённый -того, против чего вы и тестируете. Ставятся с `--force-overwrite`, потому что их +Провал здесь не проглатывается и не фатален, в отличие от имени из `packages:`, +которого конкретный фид может законно не иметь. Эти названы по URL — вы сказали +«поставь этот файл», — и об успехе без него owlab не отрапортует. Сборку он +тоже не уронит: все роутеры лаборатории собираются в одном solve buildkit, и +одна упавшая цель отменяет остальные, так что один пакет забрал бы с собой всю +лабораторию. + +Вместо этого набор ставится целиком, а если не вышло — по одному файлу, чтобы +исправные всё-таки встали. То, что так и не встало, называет `owlab up` после +своей таблицы, на этом падает `owlab test`, и это же лежит на роутере в +`/etc/owlab/extras-failed`. Напечатав такое, `owlab up` выходит с ненулевым +кодом: лаборатория работает, но это не то, что описано в вашем конфиге. + +Ставятся с `--force-overwrite`, потому что их зависимости регулярно заменяют файл, которым уже владеет стоковый образ: `luci-app-openclash` тянет `dnsmasq-full`, который ставит `/etc/init.d/dnsmasq` и сталкивается с `dnsmasq`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 385a78d..3026df3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -48,6 +48,52 @@ apk-tools 3.0.5 revalidates a cached index older than `--cache-max-age` re-downloads every APKINDEX before it resolves anything. opkg has no such policy — it reads whatever the last `opkg update` left and never asks. +### One router's `extra_packages` cancelled every other router + +**Symptom.** `owlab up` fails at the extras step of one router, and routers +that were building fine stop with it: + +``` +#33 ERROR: process "/bin/sh -c set -eu; ... opkg install --force-overwrite ..." exit code: 255 +#18 [imm2410 stage-3 2/12] ... #18 CANCELED +#24 [imm2512 stage-3 2/12] ... #24 CANCELED +owlab: build failed: exit status 1 +``` + +**Cause.** `docker compose build` puts every router in one buildkit solve, and +buildkit cancels the whole solve on the first target that fails. The extras +step exited non-zero on one staged file, so one package on one router cost the +lab. + +Both release lines do this — it is `set -eu`, not the package manager. +Measured 2026-09-04: opkg exits 255 on an unsatisfiable dependency, apk exits +27 on `unable to select packages`, and each cancelled the other routers in the +same run. There is no build-side setting to fall back on: neither `docker +compose build` nor `docker buildx bake` has a `--keep-going`. + +**Fix.** The extras step installs the set, and when that fails installs each +file on its own so the good ones still land. What is still missing is named in +the build log, recorded in `/etc/owlab/extras-failed`, reported by `owlab up` +after its table, and failed on by `owlab test`: + +``` +! owrt2410 is running WITHOUT luci-app-example_1.0_all.ipk +! Every other router built and started; only these packages are missing. +! The package manager said why during the build — one router at a time shows it again: +! owlab up --rebuild owrt2410 +``` + +`owlab up` exits non-zero when it prints that. The lab is running and every +other router is usable, but it is not what the config describes, and a script +that runs `owlab up` before its own checks must not read one as the other. + +Ask a running router directly: + +```console +$ owlab exec owrt2410 -- cat /etc/owlab/extras-failed +luci-app-example_1.0_all.ipk +``` + --- ## "The router never answers on its published port" @@ -233,7 +279,9 @@ Only on OpenWrt 24.10. `dnsmasq-full`, so three of the four routers were unaffected — the same config produced a different router on one of them. -**Fix.** `--force-overwrite`, and a failure in `extra_packages` is now fatal. +**Fix.** `--force-overwrite`, and a package in `extra_packages` that does not +install is named rather than passed over — see "One router's `extra_packages` +cancelled every other router" above for where that is reported. --- diff --git a/docs/troubleshooting_ru.md b/docs/troubleshooting_ru.md index 27b89ae..b5558a1 100644 --- a/docs/troubleshooting_ru.md +++ b/docs/troubleshooting_ru.md @@ -50,6 +50,53 @@ downloads.openwrt.org bash 5.2.37-r1 Size 473647 SHA256 20eaa220... opkg такой политики нет — он читает то, что оставил последний `opkg update`, и ничего не спрашивает. +### `extra_packages` одного роутера отменили сборку всех остальных + +**Симптом.** `owlab up` падает на шаге extras одного роутера, и вместе с ним +останавливаются роутеры, которые собирались нормально: + +``` +#33 ERROR: process "/bin/sh -c set -eu; ... opkg install --force-overwrite ..." exit code: 255 +#18 [imm2410 stage-3 2/12] ... #18 CANCELED +#24 [imm2512 stage-3 2/12] ... #24 CANCELED +owlab: build failed: exit status 1 +``` + +**Причина.** `docker compose build` кладёт все роутеры в один solve buildkit, а +buildkit отменяет весь solve на первой упавшей цели. Шаг extras выходил с +ненулевым кодом из-за одного подготовленного файла — и один пакет на одном +роутере стоил всей лаборатории. + +Так ведут себя обе линии релизов: дело в `set -eu`, а не в менеджере пакетов. +Замерено 2026-09-04: opkg выходит с 255 на неразрешимой зависимости, apk — с 27 +на `unable to select packages`, и оба раза соседние роутеры в том же запуске +получили CANCELED. Настройкой сборки это не лечится: ни у `docker compose +build`, ни у `docker buildx bake` нет `--keep-going`. + +**Что сделано.** Шаг extras ставит весь набор разом, а если это не вышло — +ставит каждый файл по отдельности, чтобы исправные всё-таки встали. То, что не +встало, названо в логе сборки, записано в `/etc/owlab/extras-failed`, сообщено +`owlab up` после его таблицы и роняет `owlab test`: + +``` +! owrt2410 is running WITHOUT luci-app-example_1.0_all.ipk +! Every other router built and started; only these packages are missing. +! The package manager said why during the build — one router at a time shows it again: +! owlab up --rebuild owrt2410 +``` + +Напечатав это, `owlab up` завершается с ненулевым кодом. Лаборатория поднята и +остальные роутеры пригодны, но это не то, что описано в конфиге, — а скрипт, +который запускает `owlab up` перед своими проверками, не должен принимать одно +за другое. + +Спросить работающий роутер напрямую: + +```console +$ owlab exec owrt2410 -- cat /etc/owlab/extras-failed +luci-app-example_1.0_all.ipk +``` + --- ## «Роутер вообще не отвечает на опубликованном порту» @@ -232,7 +279,9 @@ OpenWrt 24.10. `dnsmasq-full` из коробки, поэтому три роутера из четырёх не пострадали — один и тот же конфиг давал на одном из них другой роутер. -**Решение.** `--force-overwrite`, и провал в `extra_packages` теперь фатален. +**Решение.** `--force-overwrite`, а пакет из `extra_packages`, который не +встал, теперь называется, а не проходит незамеченным, — где именно, см. +«`extra_packages` одного роутера отменили сборку всех остальных» выше. --- diff --git a/images/Dockerfile b/images/Dockerfile index 0d22ae1..06cb9e2 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -132,11 +132,32 @@ RUN set -eu; \ # without this the same config produces a different router on OpenWrt 24.10 # than on the other three. # -# A failure here is FATAL, unlike the feed packages above. The distinction is -# that a feed name can legitimately be missing on an older release, while -# these are named by URL: the developer said "install this file", and a build -# that reports success without it hands them a router that is quietly missing -# the thing they are testing against. +# A failure here is neither swallowed like a missing feed name nor fatal. The +# distinction from the feed packages above is that a feed name can legitimately +# be absent on an older release, while these are named by URL: the developer +# said "install this file", so a build that reports success without it hands +# them a router quietly missing the thing they are testing against. +# +# Fatal was the first answer, and it was worse. `docker compose build` puts +# every router in ONE buildkit solve, and buildkit cancels the whole solve on +# the first target that fails. Measured 2026-09-04, one .ipk with an +# unsatisfiable dependency staged for one 24.10 router: +# +# #16 CANCELED (another 24.10 router, still installing feed packages) +# #13 CANCELED (a 25.12 router, still installing feed packages) +# target bad2410: failed to solve: ... exit code: 255 +# +# One package on one router cost every other router in the lab. There is no +# build-side flag to fall back on either: neither `docker compose build` nor +# `docker buildx bake` has a --keep-going, so the tolerance has to be decided +# in this RUN or not at all. +# +# What happens instead: the set is installed as one, and if that fails each +# file is installed on its own so the good ones still land. Whatever still +# fails is NAMED once per package and recorded in /etc/owlab/extras-failed — +# which `owlab up` reads back off the running router and reports after its +# table, and which `owlab test` fails on. The record is the point: a build log +# scrolls past and nobody reads it, so the fact has to outlive the build. ARG ROUTER_ID="" COPY extra/ /tmp/owlab-extra/ # @@ -173,17 +194,40 @@ COPY extra/ /tmp/owlab-extra/ # # Inside the `if`, so a build with no out-of-feed package fetches nothing, and # in this layer only, so nothing that cached before still caches. +# +# The retry installs the files in staged order, which is the order the +# developer wrote them in — see fetchExtraPackages. That is what makes a +# one-at-a-time pass usable at all: a dependent package staged after the +# package it needs installs second and resolves against a router that already +# has it. +# +# The old record is removed before anything else, and unconditionally: this +# same Dockerfile also runs on top of an already-built owlab image, and a +# failure that image recorded would otherwise be reported as this build's. RUN set -eu; \ dir="/tmp/owlab-extra/$ROUTER_ID"; \ + rm -f /etc/owlab/extras-failed; \ set -- ; \ for f in "$dir"/*; do [ -f "$f" ] && set -- "$@" "$f"; done; \ if [ "$#" -gt 0 ]; then \ for f in "$@"; do echo "owlab: installing $(basename "$f" | cut -c4-)"; done; \ if [ "$PKG_MANAGER" = "apk" ]; then \ - apk add --allow-untrusted --force-overwrite "$@"; \ + install_extras() { apk add --allow-untrusted --force-overwrite "$@"; }; \ else \ opkg update; \ - opkg install --force-overwrite "$@"; \ + install_extras() { opkg install --force-overwrite "$@"; }; \ + fi; \ + if ! install_extras "$@"; then \ + failed=""; \ + for f in "$@"; do \ + install_extras "$f" || failed="$failed $(basename "$f" | cut -c4-)"; \ + done; \ + if [ -n "$failed" ]; then \ + mkdir -p /etc/owlab; \ + for p in $failed; do echo "$p"; done > /etc/owlab/extras-failed; \ + echo "owlab: FAILED to install:$failed"; \ + echo "owlab: this router is built without it; owlab up says so at the end"; \ + fi; \ fi; \ fi; \ rm -rf /tmp/owlab-extra