From fb9c595ffeecbb8fa56f48d143a066fa16293338 Mon Sep 17 00:00:00 2001 From: Robert Nisipeanu Date: Thu, 19 Nov 2026 13:09:33 +0200 Subject: [PATCH] Deploy tool: safe mode counts each start from its first moments Safe mode turns our tools' units off after three failed starts of the head unit in a row, whatever ends each start, and the start after the third already runs without them. - safe_mode.sh's first step, at the very start of each start, counts the start as failed until it stays up 5 minutes or shuts down in order. A start that ends with no orderly shutdown at all (a power cut, a reset, a hang that is reset) counts, and so does a crash restart up to 5 minutes in: the crash handler restarts the head unit about a minute and a half after a crash, so a program crashing early in every start ends each start some 2 to 3 minutes in. - When the third failed start ends without a shutdown mark, the next start's first step writes the flag. Every tool unit that is off in safe mode starts after that step (its limits drop-in orders it after mgu-deploy-safe-mode.service). - An orderly stop of the system (systemctl reboot or poweroff), which runs no var-opt-ro.service, counts as a shutdown without a restart record: the guard's unit marks it from its ExecStop. - The count and the flag are on disk before the first step returns. - An install cut short between moving a tool's version folder aside and putting the new one in place gets the folder back, where current points, at the deploy tool's next changing run. Co-Authored-By: Claude Opus 5.5 --- deploy/CHANGELOG.md | 3 + deploy/README.md | 16 +- deploy/test/run.js | 160 +++++++++++++------ deploy/unit/mgu-deploy-safe-mode.service | 20 ++- deploy/unit/mgu-deploy.js | 40 +++-- deploy/unit/safe_mode.sh | 193 ++++++++++++++++------- 6 files changed, 302 insertions(+), 130 deletions(-) diff --git a/deploy/CHANGELOG.md b/deploy/CHANGELOG.md index 9594e7d1..002a223d 100644 --- a/deploy/CHANGELOG.md +++ b/deploy/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Safe mode counts every start as failed from its first moments until it stays up 5 minutes or shuts down in order, so starts that end in a power cut, a reset or a hang count too, and so does a crash restart up to 5 minutes into a start (the crash handler restarts the head unit about a minute and a half after a crash). Three failed starts in a row turn it on, and the start that follows already runs without our tools: their units start after the guard's first step. A `systemctl reboot` or `poweroff` counts as an orderly shutdown. +- An install cut short between moving a tool's version folder aside and putting the new one in its place gets the folder back at the deploy tool's next install or other change. + ## 0.8.25 - `uninstall` refuses to remove a component that an installed app or tool still depends on (its requirements, or a capability only that component provides), and names those components. Remove them first, or use `remove-all`, which removes them in order. diff --git a/deploy/README.md b/deploy/README.md index 2de4783c..73e7a8eb 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -576,12 +576,16 @@ services it installed stay. ### Safe mode -When the head unit restarts four times in a row within 2 minutes of -starting, by a crash or recovery restart none of our tools asked for, our -tools' units stay off from the next start: **safe mode**. BMW's units are -never touched. Units marked `"runsInSafeMode": true` in their tool's -`component.json` still run; give it to units that recover the head unit or -leave safe mode: +When three starts of the head unit end within 5 minutes, by a crash or +recovery restart none of our tools asked for or with no orderly shutdown +at all (a power cut, a reset), and no start between them stays up 5 +minutes, our tools' units stay off from the next start: **safe mode**. The +guard runs first at each start and our tools' units start after it, so the +start that follows the third failed one already runs without them. A +shutdown in order (switching the car off, the Store's or the HMI's +restart) never counts. BMW's units are never touched. Units marked +`"runsInSafeMode": true` in their tool's `component.json` still run; give +it to units that recover the head unit or leave safe mode: ```json "units": [{ "name": "acme-recovery.service", "from": "systemd/acme-recovery.service", "runsInSafeMode": true }] diff --git a/deploy/test/run.js b/deploy/test/run.js index b147273e..ab6d3d44 100644 --- a/deploy/test/run.js +++ b/deploy/test/run.js @@ -4638,6 +4638,16 @@ function testStaleFiles() { check('stale: a killed run\'s staging, window, write, temp, probe and partial files go (a never-finished tool\'s too); a live run\'s stay', r.status === 0 && left.length === 0 && fs.existsSync(path.join(dd, `write-${process.pid}-0`)), `${r.out}\n${left.join(' ')}`); check('stale: a day-old deploy/hu kit folder goes, a fresh one stays', !fs.existsSync(oldKit) && fs.existsSync(newKit)); + // an install of a version already on disk, cut short between moving that version's folder aside and putting the new one in + const versions = path.join(root.tool('fixture-tool'), 'versions'); + fs.renameSync(path.join(versions, '0.2.0'), path.join(versions, `0.2.0.old-${gone}`)); + fs.mkdirSync(path.join(versions, `0.2.0.partial-${gone}`)); + const r2 = root.run(['status', '--json']); + const r3 = root.run(['install', '--artifact', toolArtifact('0.2.1'), '--yes']); + check('stale: a version folder a cut-short install moved aside comes back where current points, at the next changing run', + r2.status === 0 && r3.status === 0 && /the version folder a stopped install moved aside is back/.test(r3.out) && + fs.existsSync(path.join(versions, '0.2.0', 'component.json')) && + fs.readdirSync(versions).every((n) => !n.includes(String(gone))), `${r3.out}\n${fs.readdirSync(versions).join(' ')}`); } /* restore refuses what it would leave broken: a component installed after @@ -5678,9 +5688,11 @@ function testServiceLimits() { '# crash handler, which would restart the head unit, a service that keeps failing stops, and\n' + '# it does not start in safe mode.\n' + '[Unit]\nStartLimitIntervalSec=600\nStartLimitBurst=20\n' + - 'ConditionPathExists=|!/var/opt/mgu-deploy/safe-mode\nConditionPathExists=|/run/mgu-deploy/safe-mode-lifted\nConditionPathExists=|!/var/opt/mgu-deploy/current/safe_mode.sh\n\n[Service]\nLimitCORE=1\n'; + 'ConditionPathExists=|!/var/opt/mgu-deploy/safe-mode\nConditionPathExists=|/run/mgu-deploy/safe-mode-lifted\nConditionPathExists=|!/var/opt/mgu-deploy/current/safe_mode.sh\n' + + 'After=mgu-deploy-safe-mode.service\n\n[Service]\nLimitCORE=1\n'; const SOCKET = '# Written by mgu-deploy for every unit a tool installs: it does not start in safe mode.\n' + - '[Unit]\nConditionPathExists=|!/var/opt/mgu-deploy/safe-mode\nConditionPathExists=|/run/mgu-deploy/safe-mode-lifted\nConditionPathExists=|!/var/opt/mgu-deploy/current/safe_mode.sh\n'; + '[Unit]\nConditionPathExists=|!/var/opt/mgu-deploy/safe-mode\nConditionPathExists=|/run/mgu-deploy/safe-mode-lifted\nConditionPathExists=|!/var/opt/mgu-deploy/current/safe_mode.sh\n' + + 'After=mgu-deploy-safe-mode.service\n'; const sys = (root, rel) => root.p(`etc/systemd/system/${rel}`); const dropIn = (root, unit) => sys(root, `${unit}.d/zz-mgu-deploy-limits.conf`); const limitsState = (res) => { try { return JSON.parse(res.stdout).service_limits; } catch (e) { return null; } }; @@ -5884,7 +5896,8 @@ function testSafeMode() { }; const flag = gp('var/opt/mgu-deploy/safe-mode'); const countFile = gp('var/opt/mgu-deploy/safe-mode-count'); - // the failed starts in a row, the last mark's start included + // the failed starts in a row: before the start the count line names, and with it + const before = () => (fs.existsSync(countFile) ? fs.readFileSync(countFile, 'utf8').trim().split(' ')[0] : null); const count = () => (fs.existsSync(countFile) ? fs.readFileSync(countFile, 'utf8').trim().split(' ')[2] : null); // how this start ends; the next start's boot step const crashAt = (s, m) => { up(s); record(m || 'recmgr', now()); guard('mark'); }; @@ -5893,32 +5906,34 @@ function testSafeMode() { start(); let r = guard('boot'); - check('safe mode: the boot step records recmgr\'s crash key and nothing else', r.status === 0 && - fs.readFileSync(gp('var/opt/mgu-deploy/safe-mode-crashes'), 'utf8') === `${bootNow()} 5\n` && - !fs.existsSync(countFile) && !fs.existsSync(flag), r.stdout + r.stderr); + check('safe mode: the boot step records recmgr\'s crash key, and counts this start as failed until it shows otherwise', + r.status === 0 && fs.readFileSync(gp('var/opt/mgu-deploy/safe-mode-crashes'), 'utf8') === `${bootNow()} 5\n` && + fs.readFileSync(countFile, 'utf8') === `0 ${bootNow()} 1\n` && !fs.existsSync(flag), r.stdout + r.stderr); + r = guard('start'); + check('safe mode: the start step again in the same start counts it once', r.status === 0 && + fs.readFileSync(countFile, 'utf8') === `0 ${bootNow()} 1\n`, fs.readFileSync(countFile, 'utf8')); let t0 = Date.now(); crashAt(60); check('safe mode: the shutdown mark records a restart recmgr asked for, with the uptime, and reads the statistics at once', fs.readFileSync(gp('var/opt/mgu-deploy/last-shutdown'), 'utf8') === `${bootNow()} crash 60 recmgr\n` && Date.now() - t0 < 1500, `${Date.now() - t0} ms`); - check('safe mode: a restart after a crash within 2 minutes counts at the shutdown', count() === '1' && + check('safe mode: a restart after a crash within 5 minutes counts', count() === '1' && fs.readFileSync(countFile, 'utf8') === `0 ${bootNow()} 1\n`); crashAt(70); check('safe mode: the mark run again in the same start counts it once', count() === '1'); next(); crashAt(150, 'crashhandler'); - check('safe mode: a crash restart after 2 minutes neither counts nor clears', count() === '1'); - next(); - crashAt(30); + check('safe mode: a restart crashhandler asks for some 2 1/2 minutes in (it restarts a minute and a half after a crash) counts', + count() === '2'); next(); - endAt(150); - check('safe mode: a switch-off or a restart the HMI or the Store asked for (a shutdown without a restart record) clears the count', + endAt(320); + check('safe mode: a switch-off or a restart the HMI or the Store asked for (a shutdown without a restart record) after 5 minutes clears the count', count() === '0'); next(); crashAt(30); next(); - endAt(25); - check('safe mode: such a shutdown within 2 minutes (a short drive, or a restart whose records were lost) neither counts nor clears', + endAt(150); + check('safe mode: such a shutdown within 5 minutes (a short drive, or a restart whose records were lost) neither counts nor clears', count() === '1'); next(); endAt(40); @@ -5954,9 +5969,6 @@ function testSafeMode() { check('safe mode: statistics that do not read as a key: no crash', count() === '2'); crashKey(12); next(); - // a power cut: no shutdown mark for this start - next(); - check('safe mode: a start with no shutdown mark (power cut) neither counts nor clears', count() === '2'); write(gp('var/opt/mgu-deploy/restart-requested'), `${bootNow()}\n`); crashAt(20); check('safe mode: a restart our tools asked for (restart-requested names the start) does not count, and clears', count() === '0'); @@ -5966,14 +5978,14 @@ function testSafeMode() { crashAt(400); check('safe mode: a crash restart after 5 minutes clears it', count() === '0'); next(); - // four in a row: decided at the fourth's shutdown, so the next start is in safe mode - [['recmgr', 30], ['crashhandler', 45], ['recmgr', 90]].forEach(([m, s]) => { crashAt(s, m); next(); }); - check('safe mode: three in a row: no safe mode', count() === '3' && !fs.existsSync(flag)); - crashAt(15); + // three in a row, each ended by a restart recmgr or crashhandler asked for: decided at the third's shutdown + [['recmgr', 30], ['crashhandler', 135]].forEach(([m, s]) => { crashAt(s, m); next(); }); + check('safe mode: two in a row: no safe mode', before() === '2' && !fs.existsSync(flag)); + crashAt(90); let f = null; try { f = JSON.parse(fs.readFileSync(flag, 'utf8')); } catch (e) { /* checked below */ } - check('safe mode: the fourth in a row turns it on at its shutdown: the flag with the time and the reason, the count from zero', - f && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/.test(f.at) && /^4 failed starts in a row, each within 120 s$/.test(f.reason) && + check('safe mode: the third in a row turns it on at its shutdown: the flag with the time and the reason, the count from zero', + f && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/.test(f.at) && /^3 failed starts in a row, each within 300 s$/.test(f.reason) && count() === '0', JSON.stringify(f)); crashAt(16); check('safe mode: a second mark of that start keeps it as it is', count() === '0' && fs.existsSync(flag)); @@ -5983,48 +5995,85 @@ function testSafeMode() { [1, 2, 3, 4].forEach(() => { crashAt(10); next(); }); check('safe mode: once on, the flag stays as it was written', fs.readFileSync(flag).equals(flagBytes)); fs.unlinkSync(flag); + endAt(320); next(); - // a start up 5 minutes clears the count (the guard's second step) + + // starts that end with no shutdown mark: power cuts, resets that skip var-opt-ro.service, a head unit that hangs and is reset crashAt(30); next(); + // the start step alone (the guard's unit as this deploy tool lays it): the flag is there before our units start + const startOnly = () => { start(); return guard('start'); }; + r = startOnly(); + check('safe mode: a start with no shutdown mark within 5 minutes counts', count() === '3' && !fs.existsSync(flag) && + fs.readFileSync(countFile, 'utf8') === `2 ${bootNow()} 3\n`, fs.readFileSync(countFile, 'utf8')); + r = startOnly(); + f = null; + try { f = JSON.parse(fs.readFileSync(flag, 'utf8')); } catch (e) { /* checked below */ } + check('safe mode: after the third in a row the next start step turns it on, before our units start, and says so', + r.status === 0 && f && /^3 failed starts in a row, each within 300 s$/.test(f.reason) && + fs.readFileSync(countFile, 'utf8') === `0 ${bootNow()} 1\n` && /safe mode: on: 3 failed starts in a row/.test(r.stdout), r.stdout + JSON.stringify(f)); + fs.unlinkSync(flag); + for (let i = 0; i < 2; i += 1) startOnly(); + check('safe mode: power cuts and resets in a row count like crash restarts', count() === '3' && !fs.existsSync(flag)); + endAt(320); + check('safe mode: and a switch-off after 5 minutes clears them', count() === '0'); + // a start up 5 minutes clears the count (the guard's up step) + startOnly(); crashAt(30); - next(); + startOnly(); up(299); t0 = Date.now(); r = upWhile(`sleep 1; ${setUp(300)}`); check('safe mode: a start that stays up 5 minutes clears the count, waiting until its uptime says so', r.status === 0 && count() === '0' && Date.now() - t0 >= 900 && /up 300 s/.test(r.stdout), r.stdout + r.stderr); + // the guard's stop at an orderly stop of the system (systemctl reboot or poweroff), no NodeStateManager shutdown + const stopAs = (sys) => { + write(gp('bin/systemctl'), `#!/bin/sh\ncase "$1" in is-system-running) echo ${sys} ;; show) echo Job= ;; esac\n`, 0o755); + return cp.spawnSync('sh', [SCRIPT, 'stop'], { encoding: 'utf8', env: Object.assign({}, env, { MGU_DEPLOY_SYSTEMCTL: gp('bin/systemctl') }), + timeout: 15000 }); + }; + startOnly(); crashAt(30); - check('safe mode: and a crash at its shutdown after it starts the count at one', count() === '1'); - next(); - - // starts that leave no shutdown mark: power cuts, resets that skip var-opt-ro.service, var-opt-ro.service failing before the mark - for (let i = 0; i < 9; i += 1) next(); - check('safe mode: starts without a shutdown mark count nothing, however many in a row', count() === '1' && !fs.existsSync(flag)); + startOnly(); + up(40); + r = stopAs('running'); + check('safe mode: the guard stopped alone (the system running on) marks nothing', r.status === 0 && count() === '2' && + !fs.readFileSync(gp('var/opt/mgu-deploy/last-shutdown'), 'utf8').startsWith(bootNow())); + r = stopAs('stopping'); + check('safe mode: an orderly stop of the system within 5 minutes is a shutdown without a record: it neither counts nor clears', + r.status === 0 && count() === '1' && fs.readFileSync(gp('var/opt/mgu-deploy/last-shutdown'), 'utf8') === `${bootNow()} normal 40\n`, r.stdout); + startOnly(); crashAt(30); - check('safe mode: and the count goes on from the last marked start', count() === '2'); - next(); - endAt(150); - next(); + r = stopAs('stopping'); + check('safe mode: a start the NodeStateManager\'s shutdown marked keeps its mark at the stop', count() === '2' && + /crash 30 recmgr$/.test(fs.readFileSync(gp('var/opt/mgu-deploy/last-shutdown'), 'utf8').trim())); + startOnly(); + up(320); + stopAs('stopping'); + check('safe mode: one after 5 minutes clears the count', count() === '0'); // corrupt or missing files never turn safe mode on + startOnly(); write(countFile, `0 x 9\n`); crashAt(30); - check('safe mode: a count at or above the threshold reads as no count: this start counts one', count() === '1' && !fs.existsSync(flag)); - next(); + check('safe mode: a count line that does not read as written reads as no count: this start counts one', count() === '1' && !fs.existsSync(flag)); + write(countFile, `0 x 9\n`); + startOnly(); + check('safe mode: nor at the next start step', count() === '1' && !fs.existsSync(flag), fs.readFileSync(countFile, 'utf8')); write(countFile, '3\n'); crashAt(30); check('safe mode: a count without its start reads as none', count() === '1' && !fs.existsSync(flag)); - next(); + startOnly(); write(countFile, 'garbage words here\n'); crashAt(30); check('safe mode: a count that does not read as written reads as none', count() === '1' && !fs.existsSync(flag)); - next(); + startOnly(); fs.unlinkSync(countFile); crashAt(30); check('safe mode: no count file: this start counts one', count() === '1' && !fs.existsSync(flag)); - next(); - write(countFile, `0 x 3\n`); + startOnly(); + write(countFile, `0 x 1\n`); + startOnly(); up(30); record('recmgr', now() - 3600); guard('mark'); @@ -6032,8 +6081,8 @@ function testSafeMode() { record('recmgr', now() + 120); guard('mark'); check('safe mode: nor one from after now', /normal 30$/.test(fs.readFileSync(gp('var/opt/mgu-deploy/last-shutdown'), 'utf8').trim())); - check('safe mode: so that start counts nothing', count() === '3' && !fs.existsSync(flag)); - next(); + check('safe mode: so that start counts nothing', count() === '1' && !fs.existsSync(flag)); + startOnly(); fs.writeFileSync(gp('proc/uptime'), 'x\n'); record('recmgr', now()); const countBefore = fs.readFileSync(countFile, 'utf8'); @@ -6043,16 +6092,19 @@ function testSafeMode() { up(30); r = guard('mark'); check('safe mode: no boot id: nothing written, exit 0', r.status === 0 && fs.readFileSync(countFile, 'utf8') === countBefore && !fs.existsSync(flag)); + r = guard('start'); + check('safe mode: nor at the start step', r.status === 0 && fs.readFileSync(countFile, 'utf8') === countBefore && !fs.existsSync(flag)); const bare = fs.mkdtempSync(path.join(T, 'bare-')); r = cp.spawnSync('sh', [SCRIPT, 'mark'], { encoding: 'utf8', env: Object.assign({}, env, { MGU_DEPLOY_VAR_OPT: bare }) }); - check('safe mode: without the deploy tool\'s folder it writes nothing', r.status === 0 && fs.readdirSync(bare).length === 0); + const r2 = cp.spawnSync('sh', [SCRIPT, 'start'], { encoding: 'utf8', env: Object.assign({}, env, { MGU_DEPLOY_VAR_OPT: bare }) }); + check('safe mode: without the deploy tool\'s folder it writes nothing', r.status === 0 && r2.status === 0 && fs.readdirSync(bare).length === 0); check('safe mode: the script leaves no temporary file', fs.readdirSync(gp('var/opt/mgu-deploy')).every((n) => !/\.tmp$|^\.crash-key/.test(n)), fs.readdirSync(gp('var/opt/mgu-deploy')).join(' ')); // a statistics read that hangs is given up after 2 s start(); write(gp('bin/sqlite3'), '#!/bin/sh\nexec sleep 30\n', 0o755); t0 = Date.now(); - r = guard('boot'); + r = guard('stats'); check('safe mode: a statistics read that does not answer is given up after 2 s, no baseline', r.status === 0 && Date.now() - t0 < 6000 && !fs.existsSync(gp('var/opt/mgu-deploy/safe-mode-crashes')), `${Date.now() - t0} ms`); // a shutdown cut during that read: the mark so far is on disk, the count as it was @@ -6099,11 +6151,17 @@ function testSafeMode() { const sock = dropIn(root, 'fixture-tool.socket'); check('safe mode: its socket: off in safe mode, nothing else', sock.includes(COND) && !/\[Service\]|StartLimit/.test(sock), sock); const dropIns = fs.readdirSync(sys(root, '')).filter((n) => /\.d$/.test(n) && fs.existsSync(sys(root, `${n}/zz-mgu-deploy-limits.conf`))); - check('safe mode: no tool unit\'s drop-in orders on the guard or wants it (nothing waits for it at boot)', dropIns.length === 3 && - dropIns.every((n) => !/mgu-deploy-safe-mode|^(Wants|After|Before|Requires|BindsTo)=/m.test(dropIn(root, n.slice(0, -2)))), - dropIns.join(' ')); + const orders = (n) => dropIn(root, n.slice(0, -2)).split('\n').filter((l) => /^(Wants|After|Before|Requires|BindsTo|Requisite)=/.test(l)); + check('safe mode: a tool unit off in safe mode starts after the guard\'s start step, and needs nothing of it; one that runs in it: no order', + dropIns.length === 3 && dropIns.every((n) => JSON.stringify(orders(n)) === + (n === 'fixture-tool-boot.service.d' ? '[]' : '["After=mgu-deploy-safe-mode.service"]')), + dropIns.map((n) => `${n}: ${orders(n).join(',')}`).join(' ')); const guardUnit = fs.readFileSync(path.join(DEPLOY, 'unit', 'mgu-deploy-safe-mode.service'), 'utf8'); - check('safe mode: the guard orders itself before nothing', !/^(Before|RequiredBy|WantedBy=(sockets|basic|sysinit|local-fs)\.target)/m.test(guardUnit)); + check('safe mode: the guard waits for nothing but /var/opt, and orders itself before nothing but the shutdown', + !/^(After|Requires|Wants)=/m.test(guardUnit) && /^RequiresMountsFor=\/var\/opt$/m.test(guardUnit) && + !/^(Before=(?!shutdown\.target$)|RequiredBy|WantedBy=(sockets|basic|sysinit|local-fs)\.target)/m.test(guardUnit) && + /^ExecStartPre=-\/bin\/sh \/var\/opt\/mgu-deploy\/current\/safe_mode\.sh start$/m.test(guardUnit) && + /^ExecStop=-\/bin\/sh \/var\/opt\/mgu-deploy\/current\/safe_mode\.sh stop$/m.test(guardUnit)); const plainLimits = dropIn(root, 'fixture-tool-boot.service'); check('safe mode: a unit that runs in safe mode: the limits alone', !plainLimits.includes('safe-mode') && plainLimits.includes('LimitCORE=1'), plainLimits); @@ -6132,11 +6190,11 @@ function testSafeMode() { // safe mode on const FLAG = root.p('var/opt/mgu-deploy/safe-mode'); - const on = '{"at": "2026-11-09T10:00:00Z", "reason": "4 failed starts in a row, each within 120 s"}\n'; + const on = '{"at": "2026-11-09T10:00:00Z", "reason": "3 failed starts in a row, each within 300 s"}\n'; write(FLAG, on); st = root.status(); check('safe mode: status --json reports it', JSON.stringify(st.safe_mode) === - JSON.stringify({ at: '2026-11-09T10:00:00Z', reason: '4 failed starts in a row, each within 120 s' }), JSON.stringify(st.safe_mode)); + JSON.stringify({ at: '2026-11-09T10:00:00Z', reason: '3 failed starts in a row, each within 300 s' }), JSON.stringify(st.safe_mode)); r = root.run(['status']); check('safe mode: plain status says so', r.stdout.includes('safe mode: on since 2026-11-09T10:00:00Z'), r.stdout); write(FLAG, 'torn'); diff --git a/deploy/unit/mgu-deploy-safe-mode.service b/deploy/unit/mgu-deploy-safe-mode.service index 8a93ee23..ce98da2b 100644 --- a/deploy/unit/mgu-deploy-safe-mode.service +++ b/deploy/unit/mgu-deploy-safe-mode.service @@ -1,17 +1,25 @@ [Unit] Description=mgu-deploy safe mode: our tools' units stay off after repeated failed starts of the head unit -# Early at each start, after /var/opt is mounted; no unit is ordered after it. +# First at each start, once /var/opt is mounted: our tools' units start after it (their drop-in +# zz-mgu-deploy-limits.conf). DefaultDependencies=no RequiresMountsFor=/var/opt +# stopped (ExecStop) at every systemd shutdown, as DefaultDependencies=yes would +Conflicts=shutdown.target +Before=shutdown.target [Service] Type=simple -# recmgr's crash statistics as they stand at the start, for the shutdown's count (safe_mode.sh). -ExecStartPre=-/bin/sh /var/opt/mgu-deploy/current/safe_mode.sh boot -# A start that stays up 5 minutes clears the count. The service stays up -# after, so recmgr never sees it end. -ExecStart=-/bin/sh -c '/bin/sh /var/opt/mgu-deploy/current/safe_mode.sh up; exec sleep 2147483647' +# This start counts as failed until it shows otherwise; after repeated failed starts, safe mode is on +# before our tools' units start. +ExecStartPre=-/bin/sh /var/opt/mgu-deploy/current/safe_mode.sh start +# recmgr's crash statistics as they stand at the start, for the shutdown's count (safe_mode.sh). A start +# that stays up 5 minutes clears the count. The service stays up after, so recmgr never sees it end. +ExecStart=-/bin/sh -c '/bin/sh /var/opt/mgu-deploy/current/safe_mode.sh stats; /bin/sh /var/opt/mgu-deploy/current/safe_mode.sh up; exec sleep 2147483647' +# An orderly stop of the system (systemctl reboot or poweroff) counts as a shutdown without a restart record. +ExecStop=-/bin/sh /var/opt/mgu-deploy/current/safe_mode.sh stop TimeoutStartSec=10 +TimeoutStopSec=10 StandardOutput=journal [Install] diff --git a/deploy/unit/mgu-deploy.js b/deploy/unit/mgu-deploy.js index 80ede902..52f0b66e 100755 --- a/deploy/unit/mgu-deploy.js +++ b/deploy/unit/mgu-deploy.js @@ -1209,9 +1209,10 @@ function recmgrWith(text, addNames, removeNames) { * StartLimitIntervalSec=0. Units of other components, stock ones included, * never get one; a drop-in of this name for a unit no installed tool lists * goes, with its folder when nothing else is in it. The same drop-in keeps - * a tool's units off in safe mode (SAFE_MODE_CONDITIONS): every service, - * socket, timer and path unit, except one whose units[] entry says - * "runsInSafeMode": true (runsInSafeMode). */ + * a tool's units off in safe mode (SAFE_MODE_CONDITIONS), and starts them + * after the guard's start step, which turns safe mode on before them + * (SAFE_MODE_AFTER): every service, socket, timer and path unit, except one + * whose units[] entry says "runsInSafeMode": true (runsInSafeMode). */ var LIMITS_NAME = 'zz-mgu-deploy-limits.conf'; var LIMITS_INTERVAL_S = 600, LIMITS_BURST = 20; var LIMITS_TEXT = ['# Written by mgu-deploy for every service a tool installs: a crash leaves no core dump for the', @@ -1225,13 +1226,13 @@ function limitsText(c, u) { if (service && !off) return LIMITS_TEXT; if (!service) { return off && /\.(socket|timer|path)$/.test(u.name) ? ['# Written by mgu-deploy for every unit a tool installs: ' + - 'it does not start in safe mode.', '[Unit]'].concat(SAFE_MODE_CONDITIONS, ['']).join('\n') : null; + 'it does not start in safe mode.', '[Unit]'].concat(SAFE_MODE_CONDITIONS, [SAFE_MODE_AFTER, '']).join('\n') : null; } return ['# Written by mgu-deploy for every service a tool installs: a crash leaves no core dump for the', '# crash handler, which would restart the head unit, a service that keeps failing stops, and', '# it does not start in safe mode.', '[Unit]', 'StartLimitIntervalSec=' + LIMITS_INTERVAL_S, 'StartLimitBurst=' + LIMITS_BURST].concat(SAFE_MODE_CONDITIONS, - ['', '[Service]', 'LimitCORE=1', '']).join('\n'); + [SAFE_MODE_AFTER, '', '[Service]', 'LimitCORE=1', '']).join('\n'); } function unitNames(c) { return (c.units || []).map(function (u) { return u.name; }); } /* The units with this tool's drop-in now. */ @@ -1274,14 +1275,13 @@ function limitsPlan(tools, only) { /* ---- safe mode ------------------------------------------------------------- */ /* After repeated failed starts of the head unit, our tools' units stay off - * (deploy/README.md, "Safe mode"): safe_mode.sh, run by a drop-in of - * var-opt-ro.service at each shutdown, counts the starts and writes - * SAFE_MODE_FILE, and mgu-deploy-safe-mode.service (ordered with none of - * our units) gives it its baseline at each start and clears the count after - * 5 minutes up; the limits drop-in keeps a unit off while the file exists. The next install that - * goes through, or `safe-mode off` (the Store's button, through the - * updater), removes it; the tools start at the next start of the head - * unit. */ + * (deploy/README.md, "Safe mode"): safe_mode.sh counts the starts and + * writes SAFE_MODE_FILE, run by mgu-deploy-safe-mode.service at each start + * (before our units, which the limits drop-in orders after it and keeps off + * while the file exists), at an orderly stop of the system, and by a drop-in + * of var-opt-ro.service at each shutdown. The next install that goes + * through, or `safe-mode off` (the Store's button, through the updater), + * removes it; the tools start at the next start of the head unit. */ var SAFE_MODE_FILE = path.join(DEPLOY_DIR, 'safe-mode'); /* in RAM: a power cut or restart mid-install leaves safe mode on */ var SAFE_MODE_LIFTED = path.join(CFG.run, 'safe-mode-lifted'); @@ -1294,6 +1294,9 @@ var SAFE_MODE_MARK = { unit: 'var-opt-ro.service', name: '40-mgu-deploy-safe-mod * the flag (triggering conditions: any one) */ var SAFE_MODE_CONDITIONS = ['ConditionPathExists=|!/var/opt/mgu-deploy/safe-mode', 'ConditionPathExists=|/run/mgu-deploy/safe-mode-lifted', 'ConditionPathExists=|!/var/opt/mgu-deploy/current/safe_mode.sh']; +/* after the guard's start step, which writes the flag when the starts before + * this one failed in a row (a unit without the guard: no order) */ +var SAFE_MODE_AFTER = 'After=' + SAFE_MODE_UNIT; /* what the guard keeps: the count and the records it is made from */ var SAFE_MODE_STATE = ['safe-mode-count', 'safe-mode-crashes', 'last-shutdown'].map(function (n) { return path.join(DEPLOY_DIR, n); }); var SAFE_MODE_LEFT = 'our tools start at the next start'; @@ -2032,7 +2035,16 @@ function cleanStale() { if (!fs.existsSync(d)) return; fs.readdirSync(d).forEach(function (n) { var m = RUN_TEMP.exec(n), pid = m && Number(m[1] || m[2] || m[3]); - if (pid && !pidAlive(pid)) rmrf(path.join(d, n)); + if (!pid || pidAlive(pid)) return; + /* a version folder a killed install moved aside (placeVersion) + * and did not replace: back in place, where current may point */ + var aside = /^(.+)\.old-\d+$/.exec(n); + if (aside && path.basename(d) === 'versions' && !lstatOrNull(path.join(d, aside[1]))) { + fs.renameSync(path.join(d, n), path.join(d, aside[1])); + say(path.join(d, aside[1]) + ': the version folder a stopped install moved aside is back'); + return; + } + rmrf(path.join(d, n)); }); }); fs.readdirSync(CFG.varOpt).forEach(function (n) { diff --git a/deploy/unit/safe_mode.sh b/deploy/unit/safe_mode.sh index 0ac870d6..0ef0972b 100755 --- a/deploy/unit/safe_mode.sh +++ b/deploy/unit/safe_mode.sh @@ -1,42 +1,56 @@ #!/bin/sh # safe_mode.sh: the deploy tool's boot-loop guard (deploy/README.md, "Safe # mode"). While /var/opt/mgu-deploy/safe-mode exists, our tools' units do -# not start (ConditionPathExists=! in their drop-in zz-mgu-deploy-limits.conf). -# The flag is written at a shutdown, so it is there before the next start. +# not start (ConditionPathExists=! in their drop-in zz-mgu-deploy-limits.conf, +# which also orders them after this guard's start step). # -# safe_mode.sh boot early at each start (mgu-deploy-safe-mode.service): -# recmgr's last crash key, the baseline for the mark -# safe_mode.sh up the same service: a start that stays up RESET_UP_S -# clears the count +# safe_mode.sh start first at each start (mgu-deploy-safe-mode.service, +# before any of our units): this start counts as failed +# until it shows otherwise, and when the starts before +# it failed THRESHOLD times in a row the flag is +# written, so our units stay off from this start on +# safe_mode.sh stats the same service: recmgr's last crash key, the +# baseline for the mark +# safe_mode.sh boot start, then stats (the step the guard's unit of an +# earlier deploy tool runs) +# safe_mode.sh up the same service: a start up UP_S clears the count +# safe_mode.sh stop the same service's stop: an orderly stop of the +# system (systemctl reboot or poweroff) counts as a +# shutdown without a restart record # safe_mode.sh mark at each shutdown through the NodeStateManager # (var-opt-ro.service, drop-in # 40-mgu-deploy-safe-mode.conf), before /var/opt goes # read-only: how this start ends, the count, and at # the THRESHOLD-th failed start in a row the flag # -# A failed start: it ends in a restart recmgr or crashhandler asked for -# within FAILED_UP_S of the start, and none of our tools asked for it -# (restart-requested, written by them before they ask, names the start). -# The restart is theirs on either record of it: the restartInfo.xml each -# writes when it asks (written in this start), or a row recmgr added to its -# crash statistics (recmgr_stats.db, table Crash, whose keys only grow; a -# row only for a failure it restarts the head unit for, written just before -# it asks) since this start's boot step, the statistics written within -# STATS_RESTART_S of the shutdown (a row has no time of its own: Timestamp -# stays empty); recmgr's startup, some 10 s into a start, deletes its -# restartInfo.xml, also one of that same start. A shutdown without either -# record (a switch-off, the HMI's or the Store's restart) clears the count -# once the start was up FAILED_UP_S; earlier, it could be a restart whose -# records were lost, and neither counts nor clears. A start up RESET_UP_S -# clears it too. A crash restart after FAILED_UP_S neither counts nor -# clears. A start without a mark (a power cut, a reset that skipped -# var-opt-ro.service, or var-opt-ro.service failing before this mark -# runs) neither counts nor clears. A state file that is missing or does -# not read as written counts nothing. +# A failed start ends within UP_S of the start, by a restart recmgr or +# crashhandler asked for or with no shutdown mark at all, and none of our +# tools asked for that restart (restart-requested, written by them before +# they ask, names the start). crashhandler restarts the head unit about a +# minute and a half after the crash it restarts it for, so a program that +# crashes early in every start ends each start some 2 to 3 minutes in. The +# start step counts each start as failed; what follows decides: +# - the start stays up UP_S: the count goes back to zero; +# - a mark at its shutdown: a restart recmgr or crashhandler asked for +# counts, on either record of it: the restartInfo.xml each writes when it +# asks (written in this start), or a row recmgr added to its crash +# statistics (recmgr_stats.db, table Crash, whose keys only grow; a row +# only for a failure it restarts the head unit for, written just before +# it asks) since this start's stats step, the statistics written within +# STATS_RESTART_S of the shutdown (a row has no time of its own: +# Timestamp stays empty); recmgr's startup, some 10 s into a start, +# deletes its restartInfo.xml, also one of that same start. A shutdown +# without either record (a switch-off, the HMI's or the Store's restart) +# neither counts nor clears; +# - an orderly stop of the system (stop) is a shutdown without a record; +# - no mark, no stop (a power cut, a reset that skipped var-opt-ro.service, +# a head unit that hangs and is reset): the start counts. +# A state file that is missing or does not read as written counts nothing. # -# Files in /var/opt/mgu-deploy, each written whole (renamed into place): +# Files in /var/opt/mgu-deploy, each written whole (renamed into place; the +# count and the flag on disk before the start step returns): # safe-mode-count " " -# safe-mode-crashes " " +# safe-mode-crashes " " # last-shutdown " crash|normal [recmgr] [crashhandler] [recmgr_stats]" # restart-requested "": a restart our tools asked for in that start # safe-mode {"at": "", "reason": ""} @@ -48,9 +62,9 @@ SIN=$VAR_OPT/public/sin BOOT_ID=${MGU_DEPLOY_BOOT_ID:-/proc/sys/kernel/random/boot_id} UPTIME=${MGU_DEPLOY_PROC:-/proc}/uptime PROC_STAT=${MGU_DEPLOY_PROC_STAT:-/proc/stat} -THRESHOLD=4 -FAILED_UP_S=120 -RESET_UP_S=300 +SYSTEMCTL=${MGU_DEPLOY_SYSTEMCTL:-systemctl} +THRESHOLD=3 +UP_S=300 # how often `up` looks at the uptime while it waits UP_POLL_S=5 # the longest a read of recmgr's statistics may take @@ -87,23 +101,57 @@ wait_up() { } is_number() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; [ "${#1}" -le 9 ]; } -# a count this guard writes: below the threshold +# a count of failed starts before a start: below the threshold is_count() { is_number "$1" && [ "$1" -lt "$THRESHOLD" ]; } # put : the text whole into the file put() { printf '%s\n' "$2" > "$1.tmp" && mv -f "$1.tmp" "$1"; } +# The count line as this guard writes it, in b, id and c: b failed starts +# before start id (below the threshold), c with it (b, b + 1 or 0); b and c +# empty when the line does not read so. +read_count() { + # shellcheck disable=SC2046 # three words + set -- $(first_line "$COUNT") + b=$1 id=$2 c=$3 + case "$id" in ''|*[!0-9A-Za-z-]*) b= c= ; return ;; esac + if ! is_count "$b" || ! is_number "$c" || { [ "$c" != "$b" ] && [ "$c" != $((b + 1)) ] && [ "$c" != 0 ]; }; then + b= c= + fi +} + +flag_on() { + [ -e "$FLAG" ] || put "$FLAG" "{\"at\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\", \"reason\": \"$1 failed starts in a row, each within $UP_S s\"}" +} + # count_set : the count, and at the # threshold the flag (the count then starts again) count_set() { c=$3 if [ "$c" -ge "$THRESHOLD" ]; then + flag_on "$c" c=0 - [ -e "$FLAG" ] || put "$FLAG" "{\"at\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\", \"reason\": \"$THRESHOLD failed starts in a row, each within $FAILED_UP_S s\"}" fi put "$COUNT" "$1 $2 $c" } +# This start counted as failed until it shows otherwise, once per start: +# the failed starts before it are what the last start's count says it ended +# with; at the threshold the flag. On disk before it returns, so a start that +# ends with no mark (a power cut, a reset) still counts. +start() { + read_count + [ "$id" = "$boot" ] && [ -n "$b" ] && return 0 + base=${c:-0} + if [ "$base" -ge "$THRESHOLD" ]; then + flag_on "$base" + say "on: $base failed starts in a row; our tools' units do not start" + base=0 + fi + put "$COUNT" "$base $boot $((base + 1))" + sync +} + # recmgr's last crash key (0 with none), read from the database opened # read-only within STATS_WAIT_S; empty without its statistics or sqlite3, # or when they cannot be read. @@ -125,15 +173,51 @@ crash_key() { return 0 } +stats() { + k=$(crash_key) + if [ -n "$k" ]; then put "$CRASHES" "$boot $k"; else rm -f "$CRASHES"; fi + [ -e "$FLAG" ] && say "on: our tools' units do not start" +} + +# whether this start has a shutdown mark already +marked() { [ "$(first_line "$LAST" | cut -d' ' -f1)" = "$boot" ]; } + +# end : how this start ends, and the count +end() { + kind=$1 up=$2 + read_count + # the count before this start: the one its start step wrote, else the last + # start's (no start step ran in this start) + if [ "$id" = "$boot" ] && [ -n "$b" ]; then + base=$b + else + base=${c:-0} + if [ "$base" -ge "$THRESHOLD" ]; then flag_on "$base"; base=0; fi + fi + if [ "$(first_line "$REQUESTED")" = "$boot" ] || [ "$up" -ge "$UP_S" ]; then + count=0 + elif [ "$kind" = crash ]; then + count=$((base + 1)) + else + count=$base + fi + count_set "$base" "$boot" "$count" +} + [ -d "$DIR" ] || exit 0 boot=$(first_line "$BOOT_ID") case "$boot" in ''|*[!0-9A-Za-z-]*) exit 0 ;; esac case "$1" in +start) + start + ;; +stats) + stats + ;; boot) - k=$(crash_key) - if [ -n "$k" ]; then put "$CRASHES" "$boot $k"; else rm -f "$CRASHES"; fi - [ -e "$FLAG" ] && say "on: our tools' units do not start" + start + stats ;; mark) up=$(uptime_s) @@ -164,30 +248,33 @@ mark) fi if [ -n "$who" ]; then kind=crash; else kind=normal; fi put "$LAST" "$boot $kind $up$who" - # the count before this start: the one written in it (a mark again in the - # same start), else the last start's - # shellcheck disable=SC2046 # three words - set -- $(first_line "$COUNT") - if [ "$2" = "$boot" ]; then base=$1; else base=$3; fi - is_count "$base" && [ -n "$2" ] || base=0 - if [ "$(first_line "$REQUESTED")" = "$boot" ]; then - count=0 - elif [ "$kind" = crash ] && [ "$up" -lt "$FAILED_UP_S" ]; then - count=$((base + 1)) - elif { [ "$kind" = normal ] && [ "$up" -ge "$FAILED_UP_S" ]; } || { [ "$kind" = crash ] && [ "$up" -ge "$RESET_UP_S" ]; }; then - count=0 - else - count=$base - fi - count_set "$base" "$boot" "$count" + end "$kind" "$up" + ;; +stop) + # an orderly stop of the system only (systemctl reboot or poweroff), not a + # stop of this unit alone; a start the NodeStateManager's shutdown marked + # keeps its mark + sys=$("$SYSTEMCTL" is-system-running 2>/dev/null) + job=$("$SYSTEMCTL" show shutdown.target -p Job 2>/dev/null) + case "$sys $job" in + stopping* | *Job=[0-9]*) ;; + *) exit 0 ;; + esac + marked && exit 0 + up=$(uptime_s) + [ -n "$up" ] || exit 0 + put "$LAST" "$boot normal $up" + end normal "$up" + sync ;; up) - wait_up "$RESET_UP_S" + wait_up "$UP_S" # shellcheck disable=SC2046 # three words set -- $(first_line "$COUNT") if [ "$*" != "0 $boot 0" ]; then put "$COUNT" "0 $boot 0" - say "up ${RESET_UP_S} s: the count starts again" + sync + say "up ${UP_S} s: the count starts again" fi ;; esac