diff --git a/DESIGN.md b/DESIGN.md index 9c00773..20489f6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -2066,11 +2066,15 @@ it started before. broker again later, and no alert goes out each time. 8. It writes the `broker_started` entry. No later step of the start can fail. -9. It starts to poll Telegram, and sends the alerts that waited. After a - power cut, the host can be up before Telegram can be reached. So when - an alert does not go out, the broker tries again: after 5 seconds, - then twice as long each time, at most 5 minutes, until the alert is - out or the broker stops. Each failed try gets one log line. +9. It starts to poll Telegram, and sends the alerts that waited. It sends + them at the same time, each on its own, so a slow try of one alert + does not delay the others. They can reach the phone in any order. + After a power cut, the host can be up before Telegram can be reached. + So when an alert does not go out, the broker tries again: after 5 + seconds, then twice as long each time, at most 5 minutes, until the + alert is out or the broker stops. Each alert has its own tries. Each + failed try gets one log line, with the number of the alert, such as + `alert=2` for the second one that waited. 10. Only then does it read the connections on the `listen` address and take requests. @@ -2086,15 +2090,20 @@ order: the `foreign_tap` entries that still wait. 3. The server: it closes the `listen` address and every connection. 4. The alerts of the start: each one gets its first try, also when the - stop began right after the start, but no try again. An alert that did - not go out is lost; the journal says so. + stop began right after the start, but no try again. The tries run at + the same time, and a try takes at most 30 seconds. So they end at + most 30 seconds after the stop began, however many alerts wait. An + alert that did not go out is lost; the journal says so. 5. The stores: they clear their tokens and session keys. 6. The `broker_stopped` entry. Then the audit log is closed. The stop takes at most 80 seconds. A request may need up to 30 seconds -to take its answer, and a try of an alert up to 30 seconds. After 80 seconds the -broker ends without the rest of the stop, and exits with 1. The example -unit gives it 90 seconds (`TimeoutStopSec=`); then systemd kills it. +to take its answer, and the alert about an audit log that cannot be +written up to 30 seconds more. The tries of the alerts of the start run +at the same time as these steps, and end within 30 seconds. After 80 +seconds the broker ends without the rest of the stop, and exits with 1. +The example unit gives it 90 seconds (`TimeoutStopSec=`); then systemd +kills it. `nodkey broker` exits with: diff --git a/cmd/nodkey/broker.go b/cmd/nodkey/broker.go index 6898458..fd6a68d 100644 --- a/cmd/nodkey/broker.go +++ b/cmd/nodkey/broker.go @@ -64,8 +64,10 @@ const ( // stopTimeout bounds the whole stop. The service waits for the // requests in progress, which may write an answer for up to 30 // seconds, and for its alert messages, up to 30 seconds more. The - // example unit sets TimeoutStopSec=90s, so the broker ends on its own - // first and logs why. + // tries of the alert messages of the start that are in progress run at + // the same time, for at most alertTimeout (sendAlerts). The example + // unit sets TimeoutStopSec=90s, so the broker ends on its own first and + // logs why. stopTimeout = 80 * time.Second ) @@ -222,8 +224,9 @@ type brokerParts struct { // alerts are the alert messages that wait until the approver can send // them: a torn line in the audit log, and stores that did not start. alerts []string - // alertsDone is closed when the alerts went out, or the sender gave - // up because the stop began. stop closes alertsStop when it begins. + // alertsDone is closed when each alert went out, or its sender gave + // up because the stop began. stop closes alertsStop when it begins; + // the senders of the alerts read it. alertsDone chan struct{} alertsStop chan struct{} @@ -352,7 +355,7 @@ func (b *brokerParts) start(ctx context.Context, path string) error { } b.started = true b.startRun(core) - b.sendAlerts() + b.sendAlerts(tg.Alert) b.serve(srv, ln) b.log.Info("nodkey broker: listening; the broker is ready", slog.String("address", ln.Addr().String())) return nil @@ -449,29 +452,42 @@ func (b *brokerParts) startRun(core *approval.Core) { b.log.Info("nodkey broker: the approver runs: it polls Telegram for the answers") } -// sendAlerts sends the alert messages that waited for the approver, one -// after the other, in a goroutine of its own. Each one gets a first try, -// also when the stop has begun; the stop waits for it. After a failed -// try, sendAlert tries again (see alertFirstWait) until the alert is out -// or the stop begins. Each failed try gets one log line; the growing wait -// limits them to one line in 5 seconds at first and one in 5 minutes -// later. An alert that did not go out before the stop is lost, and the -// journal says so. -func (b *brokerParts) sendAlerts() { +// sendAlerts sends the alert messages that waited for the approver with +// send, each in a goroutine of its own. So their tries run at the same +// time: a slow try of one alert does not delay the others, and the stop +// waits for them at most as long as one try takes, however many alerts +// wait. Each alert gets a first try, also when the stop has begun; the +// stop waits for it. After a failed try, sendAlert tries again (see +// alertFirstWait) until the alert is out or the stop begins. Each failed +// try gets one log line, with the number of the alert: its place in the +// queue, from 1. The growing wait limits the lines of each alert to one +// in 5 seconds at first and one in 5 minutes later. An alert that did not +// go out before the stop is lost, and the journal says so. +func (b *brokerParts) sendAlerts(send broker.AlertFunc) { msgs := b.alerts b.alerts = nil - b.alertsDone = make(chan struct{}) + done := make(chan struct{}) + b.alertsDone = done b.alertsStop = make(chan struct{}) + sent := make([]bool, len(msgs)) + var wg sync.WaitGroup + for i, msg := range msgs { + wg.Go(func() { + defer func() { + if recover() != nil { + b.log.Error("nodkey broker: sending an alert message panicked; the panic value is not logged", + slog.Int("alert", i+1)) + } + }() + sent[i] = b.sendAlert(send, i+1, msg) + }) + } go func() { - defer close(b.alertsDone) - defer func() { - if recover() != nil { - b.log.Error("nodkey broker: sending an alert message panicked; the panic value is not logged") - } - }() + defer close(done) + wg.Wait() lost := 0 - for _, msg := range msgs { - if !b.sendAlert(msg) { + for _, ok := range sent { + if !ok { lost++ } } @@ -482,13 +498,14 @@ func (b *brokerParts) sendAlerts() { }() } -// sendAlert tries to send msg until it is out, and reports true. After the -// stop began, it makes no new try, and reports false. -func (b *brokerParts) sendAlert(msg string) bool { +// sendAlert tries to send msg, the alert with the number n, until it is +// out, and reports true. After the stop began, it makes no new try, and +// reports false. +func (b *brokerParts) sendAlert(send broker.AlertFunc, n int, msg string) bool { wait := alertFirstWait for try := 1; ; try++ { ctx, cancel := context.WithTimeout(context.Background(), alertTimeout) - err := b.tg.Alert(ctx, msg) + err := send(ctx, msg) cancel() if err == nil { return true @@ -496,12 +513,12 @@ func (b *brokerParts) sendAlert(msg string) bool { select { case <-b.alertsStop: // Telegram's errors never hold the token or Go's error text. - b.log.Warn("nodkey broker: an alert message did not go out", slog.String("error", err.Error()), - slog.Int("try", try)) + b.log.Warn("nodkey broker: an alert message did not go out", slog.Int("alert", n), + slog.String("error", err.Error()), slog.Int("try", try)) return false default: } - b.log.Warn("nodkey broker: an alert message did not go out; the broker tries again", + b.log.Warn("nodkey broker: an alert message did not go out; the broker tries again", slog.Int("alert", n), slog.String("error", err.Error()), slog.Int("try", try), slog.Duration("next_try_in", wait)) t := time.NewTimer(wait) select { diff --git a/cmd/nodkey/broker_linux_test.go b/cmd/nodkey/broker_linux_test.go index 73aef9e..53298cd 100644 --- a/cmd/nodkey/broker_linux_test.go +++ b/cmd/nodkey/broker_linux_test.go @@ -475,6 +475,13 @@ func tearAuditLog(t *testing.T, e *brokerEnv) { t.Helper() e.signal(syscall.SIGTERM) e.mustExit(exitOK) + appendTornLine(t, e) +} + +// appendTornLine adds a torn last line to the audit log of e, as a power +// cut during a write would. The broker must have stopped. +func appendTornLine(t *testing.T, e *brokerEnv) { + t.Helper() f, err := os.OpenFile(e.auditPath, os.O_WRONLY|os.O_APPEND, 0) if err != nil { t.Fatal(err) @@ -671,6 +678,106 @@ func TestBrokerAlertTriesEndAtTheStop(t *testing.T) { } } +// threeAlertsAtTheNextStart makes the next start of the broker of e, with +// seamStoreStartFails, queue three alerts: it appends a log_recovered +// entry, as from a start that recorded a torn line and stopped before its +// broker_started entry, and then a torn line. It returns a part of the +// text of each alert: about the old torn line, the new torn line and the +// store. The broker must have stopped. +func threeAlertsAtTheNextStart(t *testing.T, e *brokerEnv) []string { + t.Helper() + appendForgedRecovery(t, e, 10, strings.Repeat("ab", 32)) + entries := e.auditEntries() + seq, ok := entries[len(entries)-1]["seq"].(float64) + if !ok { + t.Fatal("the log_recovered entry has no seq") + } + appendTornLine(t, e) + return []string{ + fmt.Sprintf("Entry %d (log_recovered)", int(seq)), + fmt.Sprintf("Entry %d (log_recovered)", int(seq)+1), + "could not start", + } +} + +// TestBrokerStopWithThreeAlertsAndNoAnswer: three alerts wait at the start, +// and Telegram takes each alert message but never answers, as in a long +// outage, so each try runs into its time limit. The first tries of the +// three alerts run at the same time. A SIGTERM right after the start: the +// stop waits for these tries, as long as one of them takes, then stops +// cleanly within its limit and writes broker_stopped. Each alert got at +// least one try, and the log says that all three are lost. Tried one after +// the other, the three tries took longer than the stop limit (issue #59); +// TestStartAlertsAtTheSameTime checks the times of the real limits. +func TestBrokerStopWithThreeAlertsAndNoAnswer(t *testing.T) { + t.Parallel() + e := startListening(t, brokerOptions{holdAlerts: true}) + e.stderr.wait(t, lineListening) + e.signal(syscall.SIGTERM) + e.mustExit(exitOK) + alerts := threeAlertsAtTheNextStart(t, e) + e.start(seamStoreStartFails, seamFastAlertTries) + e.stderr.wait(t, lineListening) + e.tg.waitHeld(len(alerts)) // the first tries, at the same time + e.signal(syscall.SIGTERM) + e.mustExit(exitOK) // a clean stop, in time + for _, alert := range alerts { + if e.tg.alertCount(alert) == 0 { + t.Errorf("the alert with %q got no try", alert) + } + } + if !e.stderr.has(`they are lost" lost=3`) { + t.Errorf("the log does not say that the three alerts are lost; its log:\n%s", e.stderr.text()) + } + got := eventNames(e.auditEntries()) + want := []string{ + "broker_started", "broker_stopped", "log_recovered", "(torn line)", "log_recovered", "broker_started", + "broker_stopped", + } + if !slices.Equal(got, want) { + t.Errorf("audit entries %q, want %q", got, want) + } +} + +// TestBrokerThreeAlertsGoOutOnce: three alerts wait at the start, and +// Telegram cannot be reached. Each alert tries again on its own. Once +// Telegram can be reached, the approver gets each alert exactly once, and +// none is lost. +func TestBrokerThreeAlertsGoOutOnce(t *testing.T) { + t.Parallel() + e := startListening(t, brokerOptions{}) + e.stderr.wait(t, lineListening) + e.signal(syscall.SIGTERM) + e.mustExit(exitOK) + alerts := threeAlertsAtTheNextStart(t, e) + e.tg.setOffline(true) + e.start(seamStoreStartFails, seamFastAlertTries) + e.stderr.wait(t, lineListening) + for n := range len(alerts) { + e.stderr.waitCount(t, fmt.Sprintf(`%s" alert=%d `, lineAlertAgain, n+1), 2) + } + if n := e.tg.alertCount(""); n != 0 { + t.Fatalf("%d alerts while Telegram cannot be reached", n) + } + e.tg.setOffline(false) + for _, alert := range alerts { + e.tg.waitAlert(alert) + } + e.signal(syscall.SIGTERM) + e.mustExit(exitOK) + for _, alert := range alerts { + if n := e.tg.alertCount(alert); n != 1 { + t.Errorf("%d alerts with %q, want 1", n, alert) + } + } + if n := e.tg.alertCount(""); n != len(alerts) { + t.Errorf("%d alerts, want %d", n, len(alerts)) + } + if e.stderr.has("they are lost") { + t.Errorf("the log says that alerts are lost; its log:\n%s", e.stderr.text()) + } +} + // appendForgedRecovery appends a log_recovered entry that chains right // onto the last entry of the audit log of e, as someone with write access // to the log could: it names a torn line that never was. diff --git a/cmd/nodkey/broker_test.go b/cmd/nodkey/broker_test.go index e50ff3a..1920bab 100644 --- a/cmd/nodkey/broker_test.go +++ b/cmd/nodkey/broker_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "crypto/ed25519" "crypto/rand" "crypto/rsa" @@ -13,7 +14,9 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" + "testing/synctest" "time" "golang.org/x/crypto/ssh" @@ -308,6 +311,179 @@ func TestStopStep(t *testing.T) { } } +// startAlerts are the alert messages of the start in the tests of +// sendAlerts. +var startAlerts = []string{"alert one", "alert two", "alert three"} + +// alertLines returns the lines of log whose message ends in msgEnd, about +// the alert with the number n. +func alertLines(log, msgEnd string, n int) []string { + var lines []string + for line := range strings.SplitSeq(log, "\n") { + if strings.Contains(line+" ", fmt.Sprintf(`%s" alert=%d `, msgEnd, n)) { + lines = append(lines, line) + } + } + return lines +} + +// TestStartAlertsAtTheSameTime: three alerts of the start, and a Telegram +// that takes each alert message but never answers, so each try runs into +// its time limit. The stop begins right after the start. The first tries +// run at the same time, so the stop waits as long as one try takes, well +// within its limit, and is clean. Each alert gets exactly one try, and one +// log line for it; one more line says that the three are lost. Tried one +// after the other, the three tries took longer than the stop limit (issue +// #59). synctest runs the real times on a fake clock, and fails if a +// goroutine is left. +func TestStartAlertsAtTheSameTime(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var logs bytes.Buffer + b := &brokerParts{log: broker.NewLogger(&logs), alerts: slices.Clone(startAlerts)} + var mu sync.Mutex + tries := make(map[string]int) + inFlight, most := 0, 0 + b.sendAlerts(func(ctx context.Context, msg string) error { + mu.Lock() + tries[msg]++ + inFlight++ + most = max(most, inFlight) + mu.Unlock() + <-ctx.Done() // Telegram does not answer + mu.Lock() + inFlight-- + mu.Unlock() + return errors.New("made-up: the call timed out") + }) + start := time.Now() + runErr, clean := b.stop(start.Add(stopTimeout)) + if took := time.Since(start); runErr != nil || !clean || took != alertTimeout { + t.Errorf("the stop: error %v, clean %v, after %v; want a clean stop after one try, %v", + runErr, clean, took, alertTimeout) + } + // After a stop that was not clean, the tries go on: the test waits + // for them, so it fails without a panic of synctest. + <-b.alertsDone + mu.Lock() + defer mu.Unlock() + if most != len(startAlerts) { + t.Errorf("%d tries at the same time, want %d", most, len(startAlerts)) + } + log := logs.String() + for i, msg := range startAlerts { + if tries[msg] != 1 { + t.Errorf("%d tries of %q, want 1", tries[msg], msg) + } + if n := len(alertLines(log, "an alert message did not go out", i+1)); n != 1 { + t.Errorf("%d lines about the failed try of alert %d, want 1", n, i+1) + } + } + if strings.Contains(log, "tries again") || !strings.Contains(log, `they are lost" lost=3`) { + t.Errorf("want no try again, and a line that says that three alerts are lost; the log:\n%s", log) + } + }) +} + +// TestStartAlertsTryAgain: Telegram cannot be reached for the first 20 +// minutes after the start. Each alert tries again on its own, with a wait +// that doubles from alertFirstWait up to alertMaxWait, and one log line +// for each failed try. Once Telegram can be reached, each alert goes out +// exactly once, and makes no try after that. The stop finds none lost. +func TestStartAlertsTryAgain(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const outage = 20 * time.Minute + var logs bytes.Buffer + b := &brokerParts{log: broker.NewLogger(&logs), alerts: slices.Clone(startAlerts)} + start := time.Now() + var mu sync.Mutex + tries := make(map[string][]bool) // for each alert: whether each try went out + b.sendAlerts(func(_ context.Context, msg string) error { + out := time.Since(start) >= outage + mu.Lock() + tries[msg] = append(tries[msg], out) + mu.Unlock() + if !out { + return errors.New("made-up: cannot connect to the server") + } + return nil + }) + time.Sleep(outage + alertMaxWait) // each alert has had its next try after the outage + if runErr, clean := b.stop(time.Now().Add(stopTimeout)); runErr != nil || !clean { + t.Errorf("the stop: error %v, clean %v; want a clean stop", runErr, clean) + } + // The wait after each failed try of one alert, in the outage. + var waits []string + for at, wait := time.Duration(0), alertFirstWait; at < outage; wait = min(2*wait, alertMaxWait) { + waits = append(waits, "next_try_in="+wait.String()) + at += wait + } + log := logs.String() + mu.Lock() + defer mu.Unlock() + for i, msg := range startAlerts { + if want := append(slices.Repeat([]bool{false}, len(waits)), true); !slices.Equal(tries[msg], want) { + t.Errorf("the tries of %q went out: %v; want %d that failed, then one that went out", + msg, tries[msg], len(waits)) + } + lines := alertLines(log, "the broker tries again", i+1) + if len(lines) != len(waits) { + t.Errorf("%d lines about failed tries of alert %d, want %d", len(lines), i+1, len(waits)) + continue + } + for j, line := range lines { + if !strings.HasSuffix(line, " "+waits[j]) { + t.Errorf("alert %d, failed try %d: the line %q does not end in %q", i+1, j+1, line, waits[j]) + } + } + } + if strings.Contains(log, "they are lost") { + t.Errorf("the log says that alerts are lost:\n%s", log) + } + }) +} + +// TestStartAlertPanics: a panic while one alert is sent ends only that +// alert. Its panic value is not logged, the other alerts still go out, and +// the stop counts the alert as lost. +func TestStartAlertPanics(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var logs bytes.Buffer + b := &brokerParts{log: broker.NewLogger(&logs), alerts: slices.Clone(startAlerts)} + var mu sync.Mutex + var out []string + b.sendAlerts(func(_ context.Context, msg string) error { + if msg == startAlerts[1] { + panic(canaryWord()) + } + mu.Lock() + defer mu.Unlock() + out = append(out, msg) + return nil + }) + if runErr, clean := b.stop(time.Now().Add(stopTimeout)); runErr != nil || !clean { + t.Errorf("the stop: error %v, clean %v; want a clean stop", runErr, clean) + } + mu.Lock() + defer mu.Unlock() + want := []string{startAlerts[0], startAlerts[2]} + slices.Sort(out) + slices.Sort(want) + if !slices.Equal(out, want) { + t.Errorf("these alerts went out: %q, want %q", out, want) + } + log := logs.String() + if len(alertLines(log, "the panic value is not logged", 2)) != 1 { + t.Errorf("no line about the panic of alert 2; the log:\n%s", log) + } + if !strings.Contains(log, `they are lost" lost=1`) { + t.Errorf("the log does not say that one alert is lost:\n%s", log) + } + if strings.Contains(log, canaryWord()) { + t.Error("the log holds the panic value") + } + }) +} + // TestUnitPreventsRestarts: the example systemd unit does not start the // broker again after the exit codes that only a person can fix: wrong // usage, a polling conflict and a bot token that Telegram refuses. diff --git a/cmd/nodkey/brokerfake_linux_test.go b/cmd/nodkey/brokerfake_linux_test.go index 7cbe195..62b18de 100644 --- a/cmd/nodkey/brokerfake_linux_test.go +++ b/cmd/nodkey/brokerfake_linux_test.go @@ -46,6 +46,7 @@ type fakeTelegram struct { markups map[int64]json.RawMessage // the buttons of each message, by message ID refusePolls int // the status of the error answer to every getUpdates; 0 for none hold chan struct{} // if set, alert messages wait until it is closed + held int // the alert messages that wait at hold now offline bool // the proxy refuses every new connection, as if Telegram were unreachable connects []string // what the proxy was asked to connect to changed chan struct{} // closed and made new on each change @@ -302,12 +303,8 @@ func (f *fakeTelegram) ServeHTTP(w http.ResponseWriter, r *http.Request) { } f.serveUpdates(w, r, p.Offset, time.Duration(p.Timeout)*time.Second) case "sendMessage": - if len(call.buttons) == 0 && hold != nil { - select { - case <-hold: - case <-r.Context().Done(): - return - } + if len(call.buttons) == 0 && hold != nil && !f.waitHold(r, hold) { + return } writeTelegram(w, http.StatusOK, message(call.msgID)) case "editMessageText", "editMessageReplyMarkup": @@ -319,6 +316,27 @@ func (f *fakeTelegram) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } +// waitHold makes the alert message of r wait until hold is closed, and +// reports true, or until the client goes away, and reports false. held +// counts the messages that wait. +func (f *fakeTelegram) waitHold(r *http.Request, hold chan struct{}) bool { + f.addHeld(1) + defer f.addHeld(-1) + select { + case <-hold: + return true + case <-r.Context().Done(): + return false + } +} + +func (f *fakeTelegram) addHeld(n int) { + f.mu.Lock() + defer f.mu.Unlock() + f.held += n + f.notifyLocked() +} + // serveUpdates answers getUpdates: the queued updates from offset on, or // none after wait, as Telegram's long polling does. func (f *fakeTelegram) serveUpdates(w http.ResponseWriter, r *http.Request, offset int64, wait time.Duration) { @@ -467,6 +485,12 @@ func (f *fakeTelegram) waitAlert(text string) string { return body } +// waitHeld waits until n alert messages wait at hold at the same time. +func (f *fakeTelegram) waitHeld(n int) { + f.t.Helper() + f.waitFor(fmt.Sprintf("%d alert messages that wait at the same time", n), func() bool { return f.held >= n }) +} + // alertCount returns the number of messages without buttons whose request // body holds text. func (f *fakeTelegram) alertCount(text string) int {