From 52700c2a7ab543275d4b88f4adc41b2648391658 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 00:46:22 +0000 Subject: [PATCH] Add opt-in keepalive requests over grpc.health.v1.Health/Check A quiet HTTP/2 connection is silent: URLSession can't send HTTP/2 PING frames, and the plugin doesn't turn on Cronet's experimental heartbeat. Anything on the path that closes idle connections can then close it, such as an ALB's idle timeout (60 s by default) or a NAT mapping. A load balancer that ends HTTP/2 would answer PINGs itself anyway. NativeHttpConfig.keepAlive (NativeKeepAlive: interval, default 15 s; timeout, default 10 s) makes a client send a Connect unary grpc.health.v1.Health/Check, with an empty HealthCheckRequest, to each origin it has calls open to, every interval, over the same URLSession or Cronet engine. It sends one request at a time per origin, starting one interval after the first call opens and stopping once none is open, and cancels a request still running after the timeout. It only keeps the connection busy: it doesn't fail calls or detect dead connections, which is left to the read idle timeout being added separately. Any response counts; one other than HTTP 200 is reported once per origin through FlutterError.onError, since it usually means the server doesn't register the health service. Keepalive requests don't reach onResponse. It's all in the shared Dart adapter, so nothing changes in the native bridges. Test server: registers grpchealth v2, reports each request's client address in X-Remote-Addr, and lists the health checks it received at GET /health-checks on the admin endpoint. Tests: nine Dart unit tests against the fake plugin, which moves to test/fake_cronet_plugin.dart; a Go test for the health service and its log; and an integration test that keepalives reach the server on the same connection as a quiet Converse stream. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BqmpXY8FMm9LGHncDYKbUX --- README.md | 36 +++- example/integration_test/eliza_test.dart | 65 +++++- lib/connect_native_http.dart | 2 +- lib/src/channel/channel_http_client.dart | 82 ++++++- lib/src/channel/keep_alive.dart | 85 ++++++++ lib/src/cronet/cronet_http_client.dart | 2 + lib/src/native_http_config.dart | 33 ++- .../url_session/url_session_http_client.dart | 2 + test/cronet_http_client_test.dart | 65 +----- test/fake_cronet_plugin.dart | 70 ++++++ test/keep_alive_test.dart | 202 ++++++++++++++++++ test_server/freeze.go | 7 +- test_server/go.mod | 1 + test_server/go.sum | 2 + test_server/health.go | 46 ++++ test_server/main.go | 15 +- test_server/server_test.go | 46 +++- 17 files changed, 673 insertions(+), 88 deletions(-) create mode 100644 lib/src/channel/keep_alive.dart create mode 100644 test/fake_cronet_plugin.dart create mode 100644 test/keep_alive_test.dart create mode 100644 test_server/health.go diff --git a/README.md b/README.md index 868b660..7b16559 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,9 @@ found or verified with the tests in this package. - **No client-side dead-connection detection.** Neither stack sends HTTP/2 pings the caller controls, and server pings stop at any load balancer that terminates HTTP/2. Idle timeouts on the path must exceed a stream's longest quiet period, or the stream must carry periodic messages. See "QUIC idle timeout vs HTTP/2 keepalive pings". +- **Register `grpc.health.v1.Health` on every server, if clients use `keepAlive`.** In connect-go: + `grpchealth.Register(server, grpchealth.NewStaticChecker())`. Route `/grpc.health.v1.Health/Check` through the + load balancer like the other procedures. See "Keepalive". **The HTTP/3 endpoint** @@ -140,6 +143,32 @@ This does nothing on the **HTTP/2 fallback** used where UDP is blocked. Neither connection that dies without the interface going down goes unnoticed until a call's deadline or an application-level heartbeat. +### Keepalive + +`NativeHttpConfig.keepAlive` keeps a quiet connection carrying traffic, so nothing on the path closes it for being +idle: a load balancer's idle timeout (an AWS ALB's is 60 s by default), a NAT mapping, or the server's own. It is +opt-in, and the same on both platforms: + +```dart +NativeHttpConfig( + http3Origins: {origin}, + keepAlive: const NativeKeepAlive(interval: Duration(seconds: 15), timeout: Duration(seconds: 10)), +) +``` + +- While a client has calls open to an origin, it sends a `grpc.health.v1.Health/Check` request there every + `interval`: a Connect unary call with an empty `HealthCheckRequest`, over the same URLSession or Cronet engine. It + takes the same connection as the calls, which the integration tests check from the server's side. +- One request at a time per origin, starting one `interval` after the first call opens and stopping once none is + open. A request that hasn't ended after `timeout` is canceled. +- It only keeps the connection busy. It doesn't fail calls or notice a dead connection, and a request on another + stream isn't data on a call's own stream, so it doesn't satisfy a read timeout on that call. Any response counts, + since the request is for traffic, not the server's health. A response other than HTTP 200 is reported once per + origin through `FlutterError.onError`, since it usually means the server doesn't register the service. +- Unlike HTTP/2 PING frames, which a load balancer that ends HTTP/2 answers itself, these requests reach the server. +- Over HTTP/3 the server's QUIC pings already keep the path busy, so keepalive mostly matters on the HTTP/2 path. +- Keepalive requests don't reach `onResponse`. + ### QUIC idle timeout vs HTTP/2 keepalive pings The two mechanisms differ in who notices a dead path, which is why server settings protect the phone over HTTP/3 but @@ -193,12 +222,13 @@ Neither platform finishes a stream on its own while the client still has its req ## Tests - **`test/`:** host unit tests, runnable on Linux CI. They cover error classification, Connect end-stream detection, - URLSession response headers, and the shared channel adapter against a fake plugin. + URLSession response headers, and the shared channel adapter, keepalive included, against a fake plugin. - **`test_server/`:** a Go server for `connectrpc.eliza.v1.ElizaService` over HTTP/2 and HTTP/3, scripted for tests (see `eliza.go`). - `-max-idle` and `-keepalive` set its QUIC idle timeout and ping interval. - `-admin-addr` serves `POST /freeze?for=10s` and `POST /unfreeze`, which drop all its UDP traffic so a test can - simulate a dead HTTP/3 path. + simulate a dead HTTP/3 path, and `GET /health-checks`, the health checks it has received. + - It serves `grpc.health.v1.Health` for keepalive, and reports each request's client address in `X-Remote-Addr`. - Its own tests (`go test ./...`) check it over HTTP/3, including the idle timeout, pings and freezing. - It creates a CA in `test_server/certs/` on first start and reuses it, so a device only has to trust it once. - With `QLOGDIR` set, it writes a qlog trace of each QUIC connection there, which shows the server's side of a @@ -223,6 +253,8 @@ Neither platform finishes a stream on its own while the client still has its req before any Alt-Svc, must reach HTTP/3 within a few fresh clients. A loaded simulator sometimes lets TCP win URLSession's race even locally, so a single fallback is logged rather than failed. + It also checks that keepalives reach the server on the same connection as a quiet stream. + The script starts the server with a 3 s idle timeout and 1 s pings. Over HTTP/3 (iOS) the tests also check two things: a quiet stream outlives the idle timeout, and a frozen path fails an open stream within it. - **Native bridge tests:** the same script runs them after the Dart tests, against the same server. Each platform's diff --git a/example/integration_test/eliza_test.dart b/example/integration_test/eliza_test.dart index c41cadc..cc74c91 100644 --- a/example/integration_test/eliza_test.dart +++ b/example/integration_test/eliza_test.dart @@ -1,6 +1,7 @@ // Runs connect_native_http's platform client against ../test_server on a device or simulator. Start it through // ../run_integration_tests.sh, which starts the server, trusts its CA on the device and passes these defines. import 'dart:async'; +import 'dart:convert'; import 'dart:io' as io; import 'package:connect_native_http/connect_native_http.dart'; @@ -26,7 +27,8 @@ const unreachableUrl = String.fromEnvironment('UNREACHABLE_URL'); /// The test server under a name its certificate does not cover. const untrustedUrl = String.fromEnvironment('UNTRUSTED_URL'); -/// The test server's plain-HTTP admin endpoint, which can freeze its HTTP/3 path (test_server/freeze.go). +/// The test server's plain-HTTP admin endpoint, which can freeze its HTTP/3 path and lists the health checks it +/// received (test_server/freeze.go). const adminUrl = String.fromEnvironment('ADMIN_URL'); /// The QUIC idle timeout the test server advertises (its -max-idle). @@ -38,10 +40,18 @@ const introduceInterval = Duration(milliseconds: 100); final class Harness { Harness._(this.native, this.eliza); - static Future create(String baseUrl, {Set http3Origins = const {}}) async { + static Future create( + String baseUrl, { + Set http3Origins = const {}, + NativeKeepAlive? keepAlive, + }) async { late final Harness harness; final native = await NativeHttpClient.create( - NativeHttpConfig(http3Origins: http3Origins, onResponse: (info) => harness.platformProtocols.add(info.protocol)), + NativeHttpConfig( + http3Origins: http3Origins, + keepAlive: keepAlive, + onResponse: (info) => harness.platformProtocols.add(info.protocol), + ), ); if (native == null) { throw StateError('no native client on this platform'); @@ -63,7 +73,13 @@ final class Harness { /// `X-Proto` of each response: the protocol the server saw. final serverProtocols = []; - void recordHeaders(Headers headers) => serverProtocols.add(headers.get('x-proto')?.join(',') ?? ''); + /// `X-Remote-Addr` of each response: the client's address as the server saw it, which identifies the connection. + final remoteAddrs = []; + + void recordHeaders(Headers headers) { + serverProtocols.add(headers.get('x-proto')?.join(',') ?? ''); + remoteAddrs.add(headers.get('x-remote-addr')?.join(',') ?? ''); + } } /// Every case finishes in a few seconds against a local server (most well under one); a hang should fail fast. @@ -81,6 +97,19 @@ Future admin(String path) async { } } +/// Every health check the test server has received, oldest first: each one's `proto` and `remoteAddr`. +Future>> healthChecks() async { + final client = io.HttpClient(); + try { + final response = await (await client.getUrl(Uri.parse('$adminUrl/health-checks'))).close(); + expect(response.statusCode, 200, reason: 'GET $adminUrl/health-checks'); + final body = await response.transform(utf8.decoder).join(); + return (jsonDecode(body) as List).cast>(); + } finally { + client.close(); + } +} + /// Opens Converse and reads the server's greeting. Nothing flows afterwards until the caller sends. Future<(StreamController, StreamIterator)> openQuietConverse(Harness harness) async { final requests = StreamController(); @@ -245,6 +274,34 @@ void main() { await requests.close(); }); + testCase('keepalives reach the server over the connection of the quiet call they keep busy', (_) async { + if (adminUrl.isEmpty) { + markTestSkipped('needs ADMIN_URL'); + return; + } + const interval = Duration(milliseconds: 500); + final kept = await Harness.create( + serverUrl, + http3Origins: {Uri.parse(serverUrl)}, + keepAlive: const NativeKeepAlive(interval: interval, timeout: Duration(seconds: 2)), + ); + addTearDown(kept.native.close); + final (requests, responses) = await openQuietConverse(kept); + final connection = kept.remoteAddrs.last; + expect(connection, isNotEmpty, reason: 'the server reports X-Remote-Addr'); + + await Future.delayed(interval * 3 + const Duration(milliseconds: 300)); + final onConnection = (await healthChecks()).where((check) => check['remoteAddr'] == connection).toList(); + // One per interval while the stream is open, after the first interval. + expect(onConnection.length, greaterThanOrEqualTo(2), reason: 'health checks from $connection: $onConnection'); + expect({for (final check in onConnection) check['proto']}, {kept.serverProtocols.last}); + + requests.add(ConverseRequest(sentence: 'still there')); + expect(await responses.moveNext(), isTrue); + expect(responses.current.sentence, 'echo: still there'); + await requests.close(); + }); + testCase('a refused connection is reported as never sent', (_) async { expect(unreachableUrl, isNotEmpty, reason: 'run with --dart-define=UNREACHABLE_URL=...'); final unreachable = await Harness.create(unreachableUrl); diff --git a/lib/connect_native_http.dart b/lib/connect_native_http.dart index a20569e..3d6eedf 100644 --- a/lib/connect_native_http.dart +++ b/lib/connect_native_http.dart @@ -2,5 +2,5 @@ library; export 'src/native_http_client.dart' show NativeHttpClient; -export 'src/native_http_config.dart' show NativeHttpConfig, NativeHttpResponseInfo; +export 'src/native_http_config.dart' show NativeHttpConfig, NativeHttpResponseInfo, NativeKeepAlive; export 'src/native_http_exception.dart' show NativeHttpException, NativeHttpFailure; diff --git a/lib/src/channel/channel_http_client.dart b/lib/src/channel/channel_http_client.dart index 1e7f1e8..3343503 100644 --- a/lib/src/channel/channel_http_client.dart +++ b/lib/src/channel/channel_http_client.dart @@ -8,6 +8,7 @@ import '../native_http_client.dart'; import '../native_http_config.dart'; import '../native_http_exception.dart'; import 'connect_end_stream.dart'; +import 'keep_alive.dart'; /// A platform plugin that speaks this file's channel protocol: `createEngine`, `shutdownEngine`, `open`, `write`, /// `end` and `cancel` method calls, and `headers`, `data`, `trailers`, `responseComplete`, `done`, `failed` and @@ -54,14 +55,29 @@ final class _Call { final response = Completer(); final body = StreamController(); final trailer = Headers(); - bool finished = false; ConnectEndStreamDetector? endStream; /// Set when the call's signal fired, so the outcome reports the signal's error rather than the cancellation. Object? abortError; + /// Called once, when the call finishes. + void Function()? onFinished; + + bool get finished => _finished; + bool _finished = false; + + void markFinished() { + if (_finished) { + return; + } + _finished = true; + final callback = onFinished; + onFinished = null; + callback?.call(); + } + void fail(Object error) { - finished = true; + markFinished(); if (!response.isCompleted) { response.completeError(error); } @@ -76,7 +92,7 @@ final class _Call { /// The server sent the whole response, so nothing more is sent. void completeResponse(String bridgeName) { - finished = true; + markFinished(); if (!response.isCompleted) { response.completeError(ConnectException(Code.internal, '$bridgeName: response ended without headers')); } @@ -233,7 +249,12 @@ final class _Dispatcher { /// A connect-dart [HttpClient] on one engine of a [PlatformBridge] plugin. base class ChannelHttpClient implements NativeHttpClient { - ChannelHttpClient(this._bridge, this._engineId, this._config); + ChannelHttpClient(this._bridge, this._engineId, this._config) { + final keepAlive = _config.keepAlive; + if (keepAlive != null) { + _keepAlive = KeepAliveMonitor(keepAlive, ping: _ping); + } + } /// Starts the bridge's event routing and creates an engine with [arguments], returning its id. static Future createEngine(PlatformBridge bridge, Map arguments) async { @@ -244,15 +265,55 @@ base class ChannelHttpClient implements NativeHttpClient { final PlatformBridge _bridge; final int _engineId; final NativeHttpConfig _config; + KeepAliveMonitor? _keepAlive; + + /// Origins whose keepalive got a response other than HTTP 200, which is reported only the first time. + final _keepAliveRefused = {}; @override - HttpClient get httpClient => _send; + HttpClient get httpClient => + (req) => _send(req, keepAlive: true); + + /// One keepalive request to [origin]. It isn't itself tracked, so it doesn't keep its own loop going, and it ends + /// quietly whatever happens: noticing a dead connection is left to the calls' own deadlines and timeouts. + Future _ping(Uri origin) async { + final keepAlive = _config.keepAlive!; + final url = origin.replace(path: healthCheckPath).toString(); + final signal = CancelableSignal(); + final timer = Timer(keepAlive.timeout, () => signal.cancel('${_bridge.name}: keepalive to $origin timed out')); + try { + final response = await _send( + HttpRequest(url, 'POST', healthCheckHeaders(keepAlive.timeout), null, signal), + keepAlive: false, + ); + await response.body.drain(); + if (response.status != 200 && _keepAliveRefused.add(origin)) { + reportError( + StateError( + '${_bridge.name}: keepalive: $url answered HTTP ${response.status}. Does the server register ' + 'grpc.health.v1.Health? The request still carried traffic.', + ), + StackTrace.current, + 'while sending a keepalive to $origin', + ); + } + } catch (_) { + // The request failed or timed out. Calls on a dead connection fail on their own. + } finally { + timer.cancel(); + } + } - Future _send(HttpRequest req) async { + /// Sends [req]. With [keepAlive], the call counts as open for [NativeHttpConfig.keepAlive] until it finishes, and + /// [NativeHttpConfig.onResponse] sees its response. + Future _send(HttpRequest req, {required bool keepAlive}) async { final dispatcher = _Dispatcher.of(_bridge); final methods = _bridge.methods; final call = _Call(Uri.parse(req.url)); - final id = dispatcher.register(call, _config.onResponse); + final id = dispatcher.register(call, keepAlive ? _config.onResponse : null); + if (keepAlive) { + call.onFinished = _keepAlive?.track(call.url); + } call.body.onCancel = () { if (!call.finished) { unawaited(dispatcher.cancel(id)); @@ -278,7 +339,7 @@ base class ChannelHttpClient implements NativeHttpClient { } catch (error) { // The plugin refused the call, or isn't there (a background isolate, say): nothing was sent. dispatcher.remove(id); - call.finished = true; + call.markFinished(); final cause = NativeHttpException(NativeHttpFailure.notSent, '${_bridge.name}: open failed: $error'); throw ConnectException(Code.unavailable, cause.message, cause: cause); } @@ -313,5 +374,8 @@ base class ChannelHttpClient implements NativeHttpClient { } @override - Future close() => _bridge.methods.invokeMethod('shutdownEngine', {'engineId': _engineId}); + Future close() { + _keepAlive?.close(); + return _bridge.methods.invokeMethod('shutdownEngine', {'engineId': _engineId}); + } } diff --git a/lib/src/channel/keep_alive.dart b/lib/src/channel/keep_alive.dart new file mode 100644 index 0000000..c07bc9f --- /dev/null +++ b/lib/src/channel/keep_alive.dart @@ -0,0 +1,85 @@ +import 'dart:async'; + +import 'package:connectrpc/connect.dart'; + +import '../native_http_config.dart'; + +/// The procedure each keepalive calls: gRPC's standard health service, which connect-go serves through +/// `grpchealth.Register`. +const healthCheckPath = '/grpc.health.v1.Health/Check'; + +/// A Connect unary request for [healthCheckPath]. Its body is an empty `HealthCheckRequest`, which asks about the +/// server as a whole and encodes to no bytes at all. +Headers healthCheckHeaders(Duration timeout) => Headers() + ..add('content-type', 'application/proto') + ..add('connect-protocol-version', '1') + ..add('connect-timeout-ms', '${timeout.inMilliseconds}'); + +/// Throws an [ArgumentError] unless [keepAlive] is `null` or has positive durations. +void validateKeepAlive(NativeKeepAlive? keepAlive) { + if (keepAlive == null) { + return; + } + if (keepAlive.interval <= Duration.zero) { + throw ArgumentError.value(keepAlive.interval, 'keepAlive.interval', 'must be positive'); + } + if (keepAlive.timeout <= Duration.zero) { + throw ArgumentError.value(keepAlive.timeout, 'keepAlive.timeout', 'must be positive'); + } +} + +/// `scheme://host:port` of [url]: the calls that share a connection. +Uri originOf(Uri url) => Uri(scheme: url.scheme, host: url.host, port: url.port); + +/// Runs one client's [NativeKeepAlive]: a loop per origin that has calls open, which ends once none are. +final class KeepAliveMonitor { + KeepAliveMonitor(this.config, {required this.ping}); + + final NativeKeepAlive config; + + /// Sends one keepalive request to [origin], and completes once it has ended, however it ended. + final Future Function(Uri origin) ping; + + final _origins = {}; + bool _closed = false; + + /// Keeps the connection to [url]'s origin busy until the returned function is called, which the caller does once + /// the call has finished. + void Function() track(Uri url) { + final origin = originOf(url); + final state = _origins.putIfAbsent(origin, _Origin.new); + final call = Object(); + state.calls.add(call); + if (!state.running && !_closed) { + state.running = true; + unawaited(_run(origin, state)); + } + return () => state.calls.remove(call); + } + + /// Stops sending. A keepalive in flight is left to finish. + void close() => _closed = true; + + Future _run(Uri origin, _Origin state) async { + try { + while (true) { + await Future.delayed(config.interval); + if (_closed || state.calls.isEmpty) { + return; + } + await ping(origin); + } + } finally { + state.running = false; + if (state.calls.isEmpty && identical(_origins[origin], state)) { + _origins.remove(origin); + } + } + } +} + +final class _Origin { + /// One token per open call. + final calls = {}; + bool running = false; +} diff --git a/lib/src/cronet/cronet_http_client.dart b/lib/src/cronet/cronet_http_client.dart index a00e89a..1d782bf 100644 --- a/lib/src/cronet/cronet_http_client.dart +++ b/lib/src/cronet/cronet_http_client.dart @@ -1,6 +1,7 @@ import 'package:flutter/services.dart'; import '../channel/channel_http_client.dart'; +import '../channel/keep_alive.dart'; import '../native_http_config.dart'; import '../native_http_exception.dart'; import 'cronet_error.dart'; @@ -32,6 +33,7 @@ final class CronetHttpClient extends ChannelHttpClient { CronetHttpClient._(int engineId, NativeHttpConfig config) : super(cronetBridge, engineId, config); static Future create(NativeHttpConfig config) async { + validateKeepAlive(config.keepAlive); final idle = config.quicIdleTimeout; if (idle != null && idle < const Duration(seconds: 1)) { throw ArgumentError.value(idle, 'quicIdleTimeout', 'Cronet takes whole seconds; use at least 1 s'); diff --git a/lib/src/native_http_config.dart b/lib/src/native_http_config.dart index 1a145f8..90014bf 100644 --- a/lib/src/native_http_config.dart +++ b/lib/src/native_http_config.dart @@ -9,10 +9,38 @@ final class NativeHttpResponseInfo { final String? protocol; } +/// Application-level keepalive: while a client has calls open to an origin, it sends a `grpc.health.v1.Health/Check` +/// request (Connect protocol, an empty `HealthCheckRequest`) every [interval] over the same platform stack, so the +/// connection carries traffic even while its calls are quiet. Register that service on every server the client +/// calls; in connect-go that is `grpchealth.Register`. +/// +/// URLSession has no way to send HTTP/2 PING frames, and this plugin doesn't turn on Cronet's (an experimental +/// option), so a quiet HTTP/2 connection is otherwise silent, and anything on the path that closes idle connections +/// can close it: a load balancer's idle timeout (an AWS ALB's is 60 s by default), a NAT mapping, or the server's own. +/// A load balancer that ends HTTP/2 also answers PINGs itself, while these requests reach the server. Over HTTP/3 the +/// server's QUIC pings already keep the path busy. +/// +/// It only keeps the connection busy: it doesn't fail calls or notice a dead connection, and a request on another +/// stream doesn't count as data on a call's own stream. Whatever the outcome, the next request waits [interval]. One +/// that hasn't ended within [timeout] is canceled. A response other than HTTP 200 is reported once per origin +/// through `FlutterError.onError`, since it usually means the server doesn't serve the health service. +/// +/// Each request is small, but on a phone a short interval keeps the radio awake while calls are open. +final class NativeKeepAlive { + const NativeKeepAlive({this.interval = const Duration(seconds: 15), this.timeout = const Duration(seconds: 10)}); + + /// The wait before each request, starting when a call opens to an origin with none open, and after each request + /// ends. Keep it well under the shortest idle timeout on the path. + final Duration interval; + + /// How long a request may take before it is canceled. + final Duration timeout; +} + /// Configuration shared by both platform clients. Server certificates are checked against the platform's trust /// store. final class NativeHttpConfig { - const NativeHttpConfig({this.http3Origins = const {}, this.quicIdleTimeout, this.onResponse}); + const NativeHttpConfig({this.http3Origins = const {}, this.quicIdleTimeout, this.onResponse, this.keepAlive}); /// `https://host:port` origins to try HTTP/3 with from the first request, before an `Alt-Svc` response header has /// advertised it. Both platforms fall back to HTTP/2 over TCP on their own when QUIC fails. @@ -32,4 +60,7 @@ final class NativeHttpConfig { /// Called with each response's headers, before its body is read. An exception it throws goes to /// `FlutterError.onError`, and the call goes on. final void Function(NativeHttpResponseInfo info)? onResponse; + + /// Application-level keepalive requests while this client has calls open. `null`, the default, sends none. + final NativeKeepAlive? keepAlive; } diff --git a/lib/src/url_session/url_session_http_client.dart b/lib/src/url_session/url_session_http_client.dart index f158e93..4fe6402 100644 --- a/lib/src/url_session/url_session_http_client.dart +++ b/lib/src/url_session/url_session_http_client.dart @@ -2,6 +2,7 @@ import 'package:connectrpc/connect.dart'; import 'package:flutter/services.dart'; import '../channel/channel_http_client.dart'; +import '../channel/keep_alive.dart'; import '../native_http_config.dart'; import '../native_http_exception.dart'; import 'ns_url_error.dart'; @@ -40,6 +41,7 @@ final class UrlSessionHttpClient extends ChannelHttpClient { UrlSessionHttpClient._(int engineId, NativeHttpConfig config) : super(urlSessionBridge, engineId, config); static Future create(NativeHttpConfig config) async { + validateKeepAlive(config.keepAlive); final engineId = await ChannelHttpClient.createEngine(urlSessionBridge, { 'http3Origins': [ for (final origin in config.http3Origins) [origin.host, origin.port], diff --git a/test/cronet_http_client_test.dart b/test/cronet_http_client_test.dart index 241f538..1b77ffb 100644 --- a/test/cronet_http_client_test.dart +++ b/test/cronet_http_client_test.dart @@ -7,70 +7,7 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; -/// Stands in for ConnectNativeHttpPlugin.kt: records method calls and lets a test emit the plugin's events. -class FakePlugin { - FakePlugin() { - _messenger.setMockMethodCallHandler(cronetMethodChannel, (call) async { - calls.add(call); - if (refusing.contains(call.method)) { - throw PlatformException(code: 'badArguments', message: '${call.method} refused'); - } - return call.method == 'createEngine' ? 7 : null; - }); - _messenger.setMockMethodCallHandler(MethodChannel(cronetEventChannel.name), (call) async => null); - } - - final calls = []; - - /// Methods the plugin answers with an error. - final refusing = {}; - - TestDefaultBinaryMessenger get _messenger => TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; - - List callsTo(String method) => calls.where((c) => c.method == method).toList(); - - int get lastCallId => callsTo('open').last.arguments['callId'] as int; - - Future emit(Map event) async { - await _messenger.handlePlatformMessage( - cronetEventChannel.name, - const StandardMethodCodec().encodeSuccessEnvelope(event), - (_) {}, - ); - } - - Future headers(int callId, {String protocol = 'quic/1+spdy/3'}) => emit({ - 'callId': callId, - 'type': 'headers', - 'status': 200, - 'protocol': protocol, - 'headers': [ - ['content-type', 'application/connect+proto'], - ], - }); - - Future data(int callId, List bytes) => - emit({'callId': callId, 'type': 'data', 'data': Uint8List.fromList(bytes)}); - - Future failed(int callId, {required bool streamReady, int? internalErrorCode}) => emit({ - 'callId': callId, - 'type': 'failed', - 'streamReady': streamReady, - 'message': 'net::ERR', - 'internalErrorCode': internalErrorCode, - }); -} - -/// Lets queued microtasks and platform-channel replies run. -Future settle() => Future.delayed(Duration.zero); - -HttpRequest request(Stream? body, {AbortSignal? signal}) => HttpRequest( - 'https://example.test/svc/Method', - 'POST', - Headers()..add('content-type', 'application/proto'), - body, - signal, -); +import 'fake_cronet_plugin.dart'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); diff --git a/test/fake_cronet_plugin.dart b/test/fake_cronet_plugin.dart new file mode 100644 index 0000000..9a31c00 --- /dev/null +++ b/test/fake_cronet_plugin.dart @@ -0,0 +1,70 @@ +import 'package:connect_native_http/src/cronet/cronet_http_client.dart'; +import 'package:connectrpc/connect.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Stands in for ConnectNativeHttpPlugin.kt: records method calls and lets a test emit the plugin's events. +class FakePlugin { + FakePlugin() { + _messenger.setMockMethodCallHandler(cronetMethodChannel, (call) async { + calls.add(call); + if (refusing.contains(call.method)) { + throw PlatformException(code: 'badArguments', message: '${call.method} refused'); + } + return call.method == 'createEngine' ? 7 : null; + }); + _messenger.setMockMethodCallHandler(MethodChannel(cronetEventChannel.name), (call) async => null); + } + + final calls = []; + + /// Methods the plugin answers with an error. + final refusing = {}; + + TestDefaultBinaryMessenger get _messenger => TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + List callsTo(String method) => calls.where((c) => c.method == method).toList(); + + int get lastCallId => callsTo('open').last.arguments['callId'] as int; + + Future emit(Map event) async { + await _messenger.handlePlatformMessage( + cronetEventChannel.name, + const StandardMethodCodec().encodeSuccessEnvelope(event), + (_) {}, + ); + } + + Future headers(int callId, {String protocol = 'quic/1+spdy/3'}) => emit({ + 'callId': callId, + 'type': 'headers', + 'status': 200, + 'protocol': protocol, + 'headers': [ + ['content-type', 'application/connect+proto'], + ], + }); + + Future data(int callId, List bytes) => + emit({'callId': callId, 'type': 'data', 'data': Uint8List.fromList(bytes)}); + + Future failed(int callId, {required bool streamReady, int? internalErrorCode}) => emit({ + 'callId': callId, + 'type': 'failed', + 'streamReady': streamReady, + 'message': 'net::ERR', + 'internalErrorCode': internalErrorCode, + }); +} + +/// Lets queued microtasks and platform-channel replies run. +Future settle() => Future.delayed(Duration.zero); + +HttpRequest request(Stream? body, {AbortSignal? signal, String url = 'https://example.test/svc/Method'}) => + HttpRequest( + url, + 'POST', + Headers()..add('content-type', 'application/proto'), + body, + signal, + ); diff --git a/test/keep_alive_test.dart b/test/keep_alive_test.dart new file mode 100644 index 0000000..0b4869a --- /dev/null +++ b/test/keep_alive_test.dart @@ -0,0 +1,202 @@ +import 'dart:async'; + +import 'package:connect_native_http/connect_native_http.dart'; +import 'package:connect_native_http/src/cronet/cronet_http_client.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'fake_cronet_plugin.dart'; + +const interval = Duration(milliseconds: 30); +const timeout = Duration(milliseconds: 60); +const keepAlive = NativeKeepAlive(interval: interval, timeout: timeout); +const healthCheck = 'https://example.test/grpc.health.v1.Health/Check'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late FakePlugin plugin; + final clients = []; + + setUp(() => plugin = FakePlugin()); + + // A client's keepalive loop would otherwise go on into the next test, against that test's fake plugin. + tearDown(() async { + for (final client in clients) { + await client.close(); + } + clients.clear(); + }); + + Future create(NativeHttpConfig config) async { + final client = await CronetHttpClient.create(config); + clients.add(client); + return client; + } + + /// The `open` arguments of each request to [url] so far. + List> opensTo(String url) => [ + for (final call in plugin.callsTo('open')) + if ((call.arguments as Map)['url'] == url) call.arguments as Map, + ]; + + List callIds(List> opens) => [for (final open in opens) open['callId']! as int]; + + Future until(String what, bool Function() condition) async { + final deadline = DateTime.now().add(const Duration(seconds: 2)); + while (!condition()) { + if (DateTime.now().isAfter(deadline)) { + fail('timed out waiting until $what; calls: ${plugin.calls}'); + } + await Future.delayed(const Duration(milliseconds: 5)); + } + } + + /// Answers call [id] with [status] and ends it. + Future answer(int id, {int status = 200}) async { + await plugin.emit({ + 'callId': id, + 'type': 'headers', + 'status': status, + 'protocol': 'h2', + 'headers': [ + ['content-type', 'application/proto'], + ], + }); + await plugin.emit({'callId': id, 'type': 'done'}); + } + + /// Opens a call to [url] whose request side stays open, and returns its id. The call must not fail: an error on + /// its response would surface as an uncaught error and fail the test. + Future openCall(CronetHttpClient client, {String url = 'https://example.test/svc/Method'}) async { + unawaited(client.httpClient(request(StreamController().stream, url: url))); + await settle(); + return callIds(opensTo(url)).last; + } + + test('sends no keepalives unless configured', () async { + final client = await create(const NativeHttpConfig()); + await openCall(client); + await Future.delayed(interval * 4); + expect(opensTo(healthCheck), isEmpty); + }); + + test('sends a Connect health check to the origin of an open call, after the interval', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + expect(opensTo(healthCheck), isEmpty, reason: 'nothing before the first interval'); + await until('a keepalive was sent', () => opensTo(healthCheck).isNotEmpty); + final open = opensTo(healthCheck).single; + expect(open['engineId'], 7); + expect(open['method'], 'POST'); + expect(open['hasBody'], isFalse, reason: 'an empty HealthCheckRequest encodes to no bytes'); + expect(open['headers'], [ + ['content-type', 'application/proto'], + ['connect-protocol-version', '1'], + ['connect-timeout-ms', '60'], + ]); + }); + + test('keeps sending while a call is open, and stops once none is', () async { + final responses = []; + final client = await create(NativeHttpConfig(keepAlive: keepAlive, onResponse: responses.add)); + final call = await openCall(client); + for (var n = 1; n <= 3; n++) { + await until('keepalive $n was sent', () => opensTo(healthCheck).length == n); + await answer(callIds(opensTo(healthCheck)).last); + } + expect(responses, isEmpty, reason: 'onResponse sees the app\'s calls only'); + + // The call ends. A keepalive already waiting for its interval may still go out, but nothing after it. + await answer(call); + await Future.delayed(interval * 2); + for (final id in callIds(opensTo(healthCheck)).skip(3)) { + await answer(id); + } + final sent = opensTo(healthCheck).length; + await Future.delayed(interval * 4); + expect(opensTo(healthCheck), hasLength(sent)); + expect(responses, hasLength(1)); + }); + + test('a keepalive with no answer is canceled after the timeout, the call carries on, and the next follows', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + final call = await openCall(client); + await until('a keepalive was sent', () => opensTo(healthCheck).isNotEmpty); + final first = callIds(opensTo(healthCheck)).single; + + await until('the keepalive was canceled', () => plugin.callsTo('cancel').isNotEmpty); + expect(plugin.callsTo('cancel').single.arguments['callId'], first); + await plugin.emit({'callId': first, 'type': 'canceled'}); + + await until('the next keepalive was sent', () => opensTo(healthCheck).length == 2); + expect(plugin.callsTo('cancel').map((c) => c.arguments['callId']), isNot(contains(call))); + }); + + test('a failed keepalive fails no call, and the next follows', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + final call = await openCall(client); + await until('a keepalive was sent', () => opensTo(healthCheck).isNotEmpty); + await plugin.failed(callIds(opensTo(healthCheck)).single, streamReady: true, internalErrorCode: -101); + await until('the next keepalive was sent', () => opensTo(healthCheck).length == 2); + expect(plugin.callsTo('cancel').map((c) => c.arguments['callId']), isNot(contains(call))); + }); + + test('one keepalive at a time per origin, however many calls are open to it', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + await openCall(client, url: 'https://example.test/svc/Other'); + await openCall(client, url: 'https://other.test:8443/svc/Method'); + await until('each origin got a keepalive', () { + return opensTo(healthCheck).isNotEmpty && + opensTo('https://other.test:8443/grpc.health.v1.Health/Check').isNotEmpty; + }); + // The first is still waiting for an answer, so no second one has gone out to the same origin. + await Future.delayed(interval * 2 ~/ 3); + expect(opensTo(healthCheck), hasLength(1)); + }); + + test('close stops keepalives', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + await client.close(); + await Future.delayed(interval * 4); + expect(opensTo(healthCheck), isEmpty); + }); + + test('rejects intervals and timeouts that are not positive', () async { + await expectLater( + CronetHttpClient.create(const NativeHttpConfig(keepAlive: NativeKeepAlive(interval: Duration.zero))), + throwsArgumentError, + ); + await expectLater( + CronetHttpClient.create(const NativeHttpConfig(keepAlive: NativeKeepAlive(timeout: Duration(seconds: -1)))), + throwsArgumentError, + ); + expect(plugin.callsTo('createEngine'), isEmpty); + }); + + group('reporting', () { + final reported = []; + late FlutterExceptionHandler? previousOnError; + + setUp(() { + reported.clear(); + previousOnError = FlutterError.onError; + FlutterError.onError = reported.add; + }); + + tearDown(() => FlutterError.onError = previousOnError); + + test('a response other than HTTP 200 is reported once per origin, and keepalives go on', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + for (var n = 1; n <= 2; n++) { + await until('keepalive $n was sent', () => opensTo(healthCheck).length == n); + await answer(callIds(opensTo(healthCheck)).last, status: 404); + } + await until('the next keepalive was sent', () => opensTo(healthCheck).length == 3); + expect(reported, hasLength(1)); + expect('${reported.single.exception}', allOf(contains('HTTP 404'), contains('grpc.health.v1.Health'))); + }); + }); +} diff --git a/test_server/freeze.go b/test_server/freeze.go index 06704ac..035f3e5 100644 --- a/test_server/freeze.go +++ b/test_server/freeze.go @@ -63,14 +63,17 @@ func (c *freezablePacketConn) SetWriteBuffer(bytes int) error { return errors.New("the underlying conn can't set its send buffer") } -// adminHandler lets tests on a device freeze the HTTP/3 path over plain HTTP: +// adminHandler lets tests on a device freeze the HTTP/3 path, and see the health checks the server received, over +// plain HTTP: // // POST /freeze?for=10s drop all UDP traffic for 10 s // POST /unfreeze stop dropping now +// GET /health-checks every health check so far, as JSON (see healthLog) // // TCP is never frozen, so HTTP/2 calls are unaffected. -func adminHandler(conn *freezablePacketConn) http.Handler { +func adminHandler(conn *freezablePacketConn, health *healthLog) http.Handler { mux := http.NewServeMux() + mux.HandleFunc("GET /health-checks", health.serveJSON) mux.HandleFunc("POST /freeze", func(w http.ResponseWriter, r *http.Request) { d, err := time.ParseDuration(r.URL.Query().Get("for")) if err != nil || d <= 0 { diff --git a/test_server/go.mod b/test_server/go.mod index 7956202..431a203 100644 --- a/test_server/go.mod +++ b/test_server/go.mod @@ -4,6 +4,7 @@ go 1.26.2 require ( connectrpc.com/connect/v2 v2.0.1 + connectrpc.com/grpchealth/v2 v2.0.0 github.com/quic-go/quic-go v0.65.0 google.golang.org/protobuf v1.36.12 ) diff --git a/test_server/go.sum b/test_server/go.sum index 6e8adb6..bcbae76 100644 --- a/test_server/go.sum +++ b/test_server/go.sum @@ -1,5 +1,7 @@ connectrpc.com/connect/v2 v2.0.1 h1:WTAycxe87UqxSM5oxS30d1dalZ+C5xz8Ec8O6zVYkHQ= connectrpc.com/connect/v2 v2.0.1/go.mod h1:N32OLrstcPUzkykFynnefFro3+xzSdOrZlK2jHmS/xo= +connectrpc.com/grpchealth/v2 v2.0.0 h1:GOBfwE25Q70Kt0QhPawK9BKsh2tP8X0EUGeIk5Sz/ag= +connectrpc.com/grpchealth/v2 v2.0.0/go.mod h1:85+eI11LlzfuyFXX/IuB/xzEfRVdJM3/RtoMgV9AEV4= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= diff --git a/test_server/health.go b/test_server/health.go new file mode 100644 index 0000000..0b19931 --- /dev/null +++ b/test_server/health.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "net/http" + "sync" + + "connectrpc.com/grpchealth/v2" +) + +// healthCheckPath is the procedure connect_native_http's keepalive calls. +const healthCheckPath = "/" + grpchealth.HealthV1ServiceName + "/Check" + +// healthLog records each health check the server receives: its protocol, and the client's address, which identifies +// the connection it came on. The integration tests read it through the admin endpoint to see that a client's +// keepalives arrived on the same connection as the call they keep busy. +type healthLog struct { + mu sync.Mutex + entries []healthLogEntry +} + +type healthLogEntry struct { + Proto string `json:"proto"` + RemoteAddr string `json:"remoteAddr"` +} + +// record wraps next, logging each request for healthCheckPath before passing it on. +func (l *healthLog) record(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == healthCheckPath { + l.mu.Lock() + l.entries = append(l.entries, healthLogEntry{Proto: r.Proto, RemoteAddr: r.RemoteAddr}) + l.mu.Unlock() + } + next.ServeHTTP(w, r) + }) +} + +// serveJSON answers with every check logged so far, oldest first. +func (l *healthLog) serveJSON(w http.ResponseWriter, _ *http.Request) { + l.mu.Lock() + entries := append([]healthLogEntry{}, l.entries...) + l.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(entries) +} diff --git a/test_server/main.go b/test_server/main.go index 10db368..b33bccf 100644 --- a/test_server/main.go +++ b/test_server/main.go @@ -8,7 +8,8 @@ // // -max-idle and -keepalive set the server's QUIC idle timeout and ping interval. QUIC uses the smaller of the two // endpoints' idle timeouts, so -max-idle also bounds how long a client takes to notice a dead path. -admin-addr -// serves a plain-HTTP endpoint that freezes the HTTP/3 path (see adminHandler). +// serves a plain-HTTP endpoint that freezes the HTTP/3 path and lists the health checks received (see +// adminHandler). package main import ( @@ -29,6 +30,7 @@ import ( "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" + "connectrpc.com/grpchealth/v2" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" "github.com/quic-go/quic-go/qlog" @@ -66,7 +68,8 @@ func main() { log.Fatal(err) } - handler := protocolHeaders(newElizaMux(), *altSvc) + health := &healthLog{} + handler := protocolHeaders(health.record(newElizaMux()), *altSvc) tlsConfig := newTLSConfig(cert) h2 := &http.Server{Addr: *addr, Handler: handler, TLSConfig: tlsConfig, ReadHeaderTimeout: 10 * time.Second} go func() { @@ -89,7 +92,7 @@ func main() { } }() if *adminAddr != "" { - admin := &http.Server{Addr: *adminAddr, Handler: adminHandler(conn), ReadHeaderTimeout: 10 * time.Second} + admin := &http.Server{Addr: *adminAddr, Handler: adminHandler(conn, health), ReadHeaderTimeout: 10 * time.Second} go func() { if err := admin.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("admin: %v", err) @@ -125,9 +128,11 @@ func newHTTP3Server(handler http.Handler, tlsConfig *tls.Config, quicConfig *qui } } +// newElizaMux serves the eliza service, and gRPC's health service for connect_native_http's keepalive. func newElizaMux() *http.ServeMux { server := connect.NewServer() elizav1connect.RegisterElizaServiceHandler(server, elizaServer{}) + grpchealth.Register(server, grpchealth.NewStaticChecker()) mux := http.NewServeMux() connecthttp.Mount(mux, server) // A 307 makes a client send the same request body again, to the path after /redirect. The body is read first: @@ -142,11 +147,13 @@ func newElizaMux() *http.ServeMux { } // protocolHeaders logs each request and reports the protocol it arrived on in X-Proto, which is how tests check -// for HTTP/3 on clients that don't expose the negotiated protocol. HTTP/2 responses also advertise HTTP/3. +// for HTTP/3 on clients that don't expose the negotiated protocol, and the client's address in X-Remote-Addr, which +// identifies the connection. HTTP/2 responses also advertise HTTP/3. func protocolHeaders(next http.Handler, altSvc string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { log.Printf("%s %s %s from %s", r.Proto, r.Method, r.URL.Path, r.RemoteAddr) w.Header().Set("X-Proto", r.Proto) + w.Header().Set("X-Remote-Addr", r.RemoteAddr) if r.ProtoMajor < 3 && altSvc != "" { w.Header().Set("Alt-Svc", altSvc) } diff --git a/test_server/server_test.go b/test_server/server_test.go index 756b383..6466477 100644 --- a/test_server/server_test.go +++ b/test_server/server_test.go @@ -1,9 +1,11 @@ package main import ( + "bytes" "context" "crypto/tls" "crypto/x509" + "encoding/json" "errors" "io" "net" @@ -11,6 +13,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "sync" "testing" "time" @@ -168,7 +171,7 @@ func TestFrozenPathFailsOpenStreamWithinIdleTimeout(t *testing.T) { func TestAdminFreezeAndUnfreeze(t *testing.T) { conn := &freezablePacketConn{} - handler := adminHandler(conn) + handler := adminHandler(conn, &healthLog{}) post := func(target string) int { w := httptest.NewRecorder() handler.ServeHTTP(w, httptest.NewRequest(http.MethodPost, target, nil)) @@ -381,3 +384,44 @@ func TestAltSvcOnlyOnTCPResponses(t *testing.T) { } } } + +func TestHealthCheckIsServedAndLogged(t *testing.T) { + health := &healthLog{} + handler := protocolHeaders(health.record(newElizaMux()), "") + post := func(path string) *httptest.ResponseRecorder { + // An empty body: a HealthCheckRequest for the whole server, or an empty SayRequest. + req := httptest.NewRequest(http.MethodPost, path, nil) + req.Header.Set("Content-Type", "application/proto") + req.Header.Set("Connect-Protocol-Version", "1") + req.RemoteAddr = "192.0.2.1:1234" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w + } + + w := post(healthCheckPath) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body) + } + // A HealthCheckResponse with status SERVING: field 1, varint 1. + if got, want := w.Body.Bytes(), []byte{0x08, 0x01}; !bytes.Equal(got, want) { + t.Errorf("body = %x, want %x", got, want) + } + if got := w.Header().Get("X-Remote-Addr"); got != "192.0.2.1:1234" { + t.Errorf("X-Remote-Addr = %q", got) + } + if code := post("/connectrpc.eliza.v1.ElizaService/Say").Code; code != http.StatusOK { + t.Fatalf("Say: status %d", code) + } + + admin := httptest.NewRecorder() + adminHandler(&freezablePacketConn{}, health).ServeHTTP(admin, httptest.NewRequest(http.MethodGet, "/health-checks", nil)) + var entries []healthLogEntry + if err := json.Unmarshal(admin.Body.Bytes(), &entries); err != nil { + t.Fatalf("%v: %s", err, admin.Body) + } + want := []healthLogEntry{{Proto: "HTTP/1.1", RemoteAddr: "192.0.2.1:1234"}} + if !slices.Equal(entries, want) { + t.Errorf("health checks = %+v, want %+v (the Say call isn't one)", entries, want) + } +}