diff --git a/README.md b/README.md index 868b660..7b16559 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,9 @@ found or verified with the tests in this package. - **No client-side dead-connection detection.** Neither stack sends HTTP/2 pings the caller controls, and server pings stop at any load balancer that terminates HTTP/2. Idle timeouts on the path must exceed a stream's longest quiet period, or the stream must carry periodic messages. See "QUIC idle timeout vs HTTP/2 keepalive pings". +- **Register `grpc.health.v1.Health` on every server, if clients use `keepAlive`.** In connect-go: + `grpchealth.Register(server, grpchealth.NewStaticChecker())`. Route `/grpc.health.v1.Health/Check` through the + load balancer like the other procedures. See "Keepalive". **The HTTP/3 endpoint** @@ -140,6 +143,32 @@ This does nothing on the **HTTP/2 fallback** used where UDP is blocked. Neither connection that dies without the interface going down goes unnoticed until a call's deadline or an application-level heartbeat. +### Keepalive + +`NativeHttpConfig.keepAlive` keeps a quiet connection carrying traffic, so nothing on the path closes it for being +idle: a load balancer's idle timeout (an AWS ALB's is 60 s by default), a NAT mapping, or the server's own. It is +opt-in, and the same on both platforms: + +```dart +NativeHttpConfig( + http3Origins: {origin}, + keepAlive: const NativeKeepAlive(interval: Duration(seconds: 15), timeout: Duration(seconds: 10)), +) +``` + +- While a client has calls open to an origin, it sends a `grpc.health.v1.Health/Check` request there every + `interval`: a Connect unary call with an empty `HealthCheckRequest`, over the same URLSession or Cronet engine. It + takes the same connection as the calls, which the integration tests check from the server's side. +- One request at a time per origin, starting one `interval` after the first call opens and stopping once none is + open. A request that hasn't ended after `timeout` is canceled. +- It only keeps the connection busy. It doesn't fail calls or notice a dead connection, and a request on another + stream isn't data on a call's own stream, so it doesn't satisfy a read timeout on that call. Any response counts, + since the request is for traffic, not the server's health. A response other than HTTP 200 is reported once per + origin through `FlutterError.onError`, since it usually means the server doesn't register the service. +- Unlike HTTP/2 PING frames, which a load balancer that ends HTTP/2 answers itself, these requests reach the server. +- Over HTTP/3 the server's QUIC pings already keep the path busy, so keepalive mostly matters on the HTTP/2 path. +- Keepalive requests don't reach `onResponse`. + ### QUIC idle timeout vs HTTP/2 keepalive pings The two mechanisms differ in who notices a dead path, which is why server settings protect the phone over HTTP/3 but @@ -193,12 +222,13 @@ Neither platform finishes a stream on its own while the client still has its req ## Tests - **`test/`:** host unit tests, runnable on Linux CI. They cover error classification, Connect end-stream detection, - URLSession response headers, and the shared channel adapter against a fake plugin. + URLSession response headers, and the shared channel adapter, keepalive included, against a fake plugin. - **`test_server/`:** a Go server for `connectrpc.eliza.v1.ElizaService` over HTTP/2 and HTTP/3, scripted for tests (see `eliza.go`). - `-max-idle` and `-keepalive` set its QUIC idle timeout and ping interval. - `-admin-addr` serves `POST /freeze?for=10s` and `POST /unfreeze`, which drop all its UDP traffic so a test can - simulate a dead HTTP/3 path. + simulate a dead HTTP/3 path, and `GET /health-checks`, the health checks it has received. + - It serves `grpc.health.v1.Health` for keepalive, and reports each request's client address in `X-Remote-Addr`. - Its own tests (`go test ./...`) check it over HTTP/3, including the idle timeout, pings and freezing. - It creates a CA in `test_server/certs/` on first start and reuses it, so a device only has to trust it once. - With `QLOGDIR` set, it writes a qlog trace of each QUIC connection there, which shows the server's side of a @@ -223,6 +253,8 @@ Neither platform finishes a stream on its own while the client still has its req before any Alt-Svc, must reach HTTP/3 within a few fresh clients. A loaded simulator sometimes lets TCP win URLSession's race even locally, so a single fallback is logged rather than failed. + It also checks that keepalives reach the server on the same connection as a quiet stream. + The script starts the server with a 3 s idle timeout and 1 s pings. Over HTTP/3 (iOS) the tests also check two things: a quiet stream outlives the idle timeout, and a frozen path fails an open stream within it. - **Native bridge tests:** the same script runs them after the Dart tests, against the same server. Each platform's diff --git a/example/integration_test/eliza_test.dart b/example/integration_test/eliza_test.dart index c41cadc..cc74c91 100644 --- a/example/integration_test/eliza_test.dart +++ b/example/integration_test/eliza_test.dart @@ -1,6 +1,7 @@ // Runs connect_native_http's platform client against ../test_server on a device or simulator. Start it through // ../run_integration_tests.sh, which starts the server, trusts its CA on the device and passes these defines. import 'dart:async'; +import 'dart:convert'; import 'dart:io' as io; import 'package:connect_native_http/connect_native_http.dart'; @@ -26,7 +27,8 @@ const unreachableUrl = String.fromEnvironment('UNREACHABLE_URL'); /// The test server under a name its certificate does not cover. const untrustedUrl = String.fromEnvironment('UNTRUSTED_URL'); -/// The test server's plain-HTTP admin endpoint, which can freeze its HTTP/3 path (test_server/freeze.go). +/// The test server's plain-HTTP admin endpoint, which can freeze its HTTP/3 path and lists the health checks it +/// received (test_server/freeze.go). const adminUrl = String.fromEnvironment('ADMIN_URL'); /// The QUIC idle timeout the test server advertises (its -max-idle). @@ -38,10 +40,18 @@ const introduceInterval = Duration(milliseconds: 100); final class Harness { Harness._(this.native, this.eliza); - static Future create(String baseUrl, {Set http3Origins = const {}}) async { + static Future create( + String baseUrl, { + Set http3Origins = const {}, + NativeKeepAlive? keepAlive, + }) async { late final Harness harness; final native = await NativeHttpClient.create( - NativeHttpConfig(http3Origins: http3Origins, onResponse: (info) => harness.platformProtocols.add(info.protocol)), + NativeHttpConfig( + http3Origins: http3Origins, + keepAlive: keepAlive, + onResponse: (info) => harness.platformProtocols.add(info.protocol), + ), ); if (native == null) { throw StateError('no native client on this platform'); @@ -63,7 +73,13 @@ final class Harness { /// `X-Proto` of each response: the protocol the server saw. final serverProtocols = []; - void recordHeaders(Headers headers) => serverProtocols.add(headers.get('x-proto')?.join(',') ?? ''); + /// `X-Remote-Addr` of each response: the client's address as the server saw it, which identifies the connection. + final remoteAddrs = []; + + void recordHeaders(Headers headers) { + serverProtocols.add(headers.get('x-proto')?.join(',') ?? ''); + remoteAddrs.add(headers.get('x-remote-addr')?.join(',') ?? ''); + } } /// Every case finishes in a few seconds against a local server (most well under one); a hang should fail fast. @@ -81,6 +97,19 @@ Future admin(String path) async { } } +/// Every health check the test server has received, oldest first: each one's `proto` and `remoteAddr`. +Future>> healthChecks() async { + final client = io.HttpClient(); + try { + final response = await (await client.getUrl(Uri.parse('$adminUrl/health-checks'))).close(); + expect(response.statusCode, 200, reason: 'GET $adminUrl/health-checks'); + final body = await response.transform(utf8.decoder).join(); + return (jsonDecode(body) as List).cast>(); + } finally { + client.close(); + } +} + /// Opens Converse and reads the server's greeting. Nothing flows afterwards until the caller sends. Future<(StreamController, StreamIterator)> openQuietConverse(Harness harness) async { final requests = StreamController(); @@ -245,6 +274,34 @@ void main() { await requests.close(); }); + testCase('keepalives reach the server over the connection of the quiet call they keep busy', (_) async { + if (adminUrl.isEmpty) { + markTestSkipped('needs ADMIN_URL'); + return; + } + const interval = Duration(milliseconds: 500); + final kept = await Harness.create( + serverUrl, + http3Origins: {Uri.parse(serverUrl)}, + keepAlive: const NativeKeepAlive(interval: interval, timeout: Duration(seconds: 2)), + ); + addTearDown(kept.native.close); + final (requests, responses) = await openQuietConverse(kept); + final connection = kept.remoteAddrs.last; + expect(connection, isNotEmpty, reason: 'the server reports X-Remote-Addr'); + + await Future.delayed(interval * 3 + const Duration(milliseconds: 300)); + final onConnection = (await healthChecks()).where((check) => check['remoteAddr'] == connection).toList(); + // One per interval while the stream is open, after the first interval. + expect(onConnection.length, greaterThanOrEqualTo(2), reason: 'health checks from $connection: $onConnection'); + expect({for (final check in onConnection) check['proto']}, {kept.serverProtocols.last}); + + requests.add(ConverseRequest(sentence: 'still there')); + expect(await responses.moveNext(), isTrue); + expect(responses.current.sentence, 'echo: still there'); + await requests.close(); + }); + testCase('a refused connection is reported as never sent', (_) async { expect(unreachableUrl, isNotEmpty, reason: 'run with --dart-define=UNREACHABLE_URL=...'); final unreachable = await Harness.create(unreachableUrl); diff --git a/lib/connect_native_http.dart b/lib/connect_native_http.dart index a20569e..3d6eedf 100644 --- a/lib/connect_native_http.dart +++ b/lib/connect_native_http.dart @@ -2,5 +2,5 @@ library; export 'src/native_http_client.dart' show NativeHttpClient; -export 'src/native_http_config.dart' show NativeHttpConfig, NativeHttpResponseInfo; +export 'src/native_http_config.dart' show NativeHttpConfig, NativeHttpResponseInfo, NativeKeepAlive; export 'src/native_http_exception.dart' show NativeHttpException, NativeHttpFailure; diff --git a/lib/src/channel/channel_http_client.dart b/lib/src/channel/channel_http_client.dart index 1e7f1e8..3343503 100644 --- a/lib/src/channel/channel_http_client.dart +++ b/lib/src/channel/channel_http_client.dart @@ -8,6 +8,7 @@ import '../native_http_client.dart'; import '../native_http_config.dart'; import '../native_http_exception.dart'; import 'connect_end_stream.dart'; +import 'keep_alive.dart'; /// A platform plugin that speaks this file's channel protocol: `createEngine`, `shutdownEngine`, `open`, `write`, /// `end` and `cancel` method calls, and `headers`, `data`, `trailers`, `responseComplete`, `done`, `failed` and @@ -54,14 +55,29 @@ final class _Call { final response = Completer(); final body = StreamController(); final trailer = Headers(); - bool finished = false; ConnectEndStreamDetector? endStream; /// Set when the call's signal fired, so the outcome reports the signal's error rather than the cancellation. Object? abortError; + /// Called once, when the call finishes. + void Function()? onFinished; + + bool get finished => _finished; + bool _finished = false; + + void markFinished() { + if (_finished) { + return; + } + _finished = true; + final callback = onFinished; + onFinished = null; + callback?.call(); + } + void fail(Object error) { - finished = true; + markFinished(); if (!response.isCompleted) { response.completeError(error); } @@ -76,7 +92,7 @@ final class _Call { /// The server sent the whole response, so nothing more is sent. void completeResponse(String bridgeName) { - finished = true; + markFinished(); if (!response.isCompleted) { response.completeError(ConnectException(Code.internal, '$bridgeName: response ended without headers')); } @@ -233,7 +249,12 @@ final class _Dispatcher { /// A connect-dart [HttpClient] on one engine of a [PlatformBridge] plugin. base class ChannelHttpClient implements NativeHttpClient { - ChannelHttpClient(this._bridge, this._engineId, this._config); + ChannelHttpClient(this._bridge, this._engineId, this._config) { + final keepAlive = _config.keepAlive; + if (keepAlive != null) { + _keepAlive = KeepAliveMonitor(keepAlive, ping: _ping); + } + } /// Starts the bridge's event routing and creates an engine with [arguments], returning its id. static Future createEngine(PlatformBridge bridge, Map arguments) async { @@ -244,15 +265,55 @@ base class ChannelHttpClient implements NativeHttpClient { final PlatformBridge _bridge; final int _engineId; final NativeHttpConfig _config; + KeepAliveMonitor? _keepAlive; + + /// Origins whose keepalive got a response other than HTTP 200, which is reported only the first time. + final _keepAliveRefused = {}; @override - HttpClient get httpClient => _send; + HttpClient get httpClient => + (req) => _send(req, keepAlive: true); + + /// One keepalive request to [origin]. It isn't itself tracked, so it doesn't keep its own loop going, and it ends + /// quietly whatever happens: noticing a dead connection is left to the calls' own deadlines and timeouts. + Future _ping(Uri origin) async { + final keepAlive = _config.keepAlive!; + final url = origin.replace(path: healthCheckPath).toString(); + final signal = CancelableSignal(); + final timer = Timer(keepAlive.timeout, () => signal.cancel('${_bridge.name}: keepalive to $origin timed out')); + try { + final response = await _send( + HttpRequest(url, 'POST', healthCheckHeaders(keepAlive.timeout), null, signal), + keepAlive: false, + ); + await response.body.drain(); + if (response.status != 200 && _keepAliveRefused.add(origin)) { + reportError( + StateError( + '${_bridge.name}: keepalive: $url answered HTTP ${response.status}. Does the server register ' + 'grpc.health.v1.Health? The request still carried traffic.', + ), + StackTrace.current, + 'while sending a keepalive to $origin', + ); + } + } catch (_) { + // The request failed or timed out. Calls on a dead connection fail on their own. + } finally { + timer.cancel(); + } + } - Future _send(HttpRequest req) async { + /// Sends [req]. With [keepAlive], the call counts as open for [NativeHttpConfig.keepAlive] until it finishes, and + /// [NativeHttpConfig.onResponse] sees its response. + Future _send(HttpRequest req, {required bool keepAlive}) async { final dispatcher = _Dispatcher.of(_bridge); final methods = _bridge.methods; final call = _Call(Uri.parse(req.url)); - final id = dispatcher.register(call, _config.onResponse); + final id = dispatcher.register(call, keepAlive ? _config.onResponse : null); + if (keepAlive) { + call.onFinished = _keepAlive?.track(call.url); + } call.body.onCancel = () { if (!call.finished) { unawaited(dispatcher.cancel(id)); @@ -278,7 +339,7 @@ base class ChannelHttpClient implements NativeHttpClient { } catch (error) { // The plugin refused the call, or isn't there (a background isolate, say): nothing was sent. dispatcher.remove(id); - call.finished = true; + call.markFinished(); final cause = NativeHttpException(NativeHttpFailure.notSent, '${_bridge.name}: open failed: $error'); throw ConnectException(Code.unavailable, cause.message, cause: cause); } @@ -313,5 +374,8 @@ base class ChannelHttpClient implements NativeHttpClient { } @override - Future close() => _bridge.methods.invokeMethod('shutdownEngine', {'engineId': _engineId}); + Future close() { + _keepAlive?.close(); + return _bridge.methods.invokeMethod('shutdownEngine', {'engineId': _engineId}); + } } diff --git a/lib/src/channel/keep_alive.dart b/lib/src/channel/keep_alive.dart new file mode 100644 index 0000000..c07bc9f --- /dev/null +++ b/lib/src/channel/keep_alive.dart @@ -0,0 +1,85 @@ +import 'dart:async'; + +import 'package:connectrpc/connect.dart'; + +import '../native_http_config.dart'; + +/// The procedure each keepalive calls: gRPC's standard health service, which connect-go serves through +/// `grpchealth.Register`. +const healthCheckPath = '/grpc.health.v1.Health/Check'; + +/// A Connect unary request for [healthCheckPath]. Its body is an empty `HealthCheckRequest`, which asks about the +/// server as a whole and encodes to no bytes at all. +Headers healthCheckHeaders(Duration timeout) => Headers() + ..add('content-type', 'application/proto') + ..add('connect-protocol-version', '1') + ..add('connect-timeout-ms', '${timeout.inMilliseconds}'); + +/// Throws an [ArgumentError] unless [keepAlive] is `null` or has positive durations. +void validateKeepAlive(NativeKeepAlive? keepAlive) { + if (keepAlive == null) { + return; + } + if (keepAlive.interval <= Duration.zero) { + throw ArgumentError.value(keepAlive.interval, 'keepAlive.interval', 'must be positive'); + } + if (keepAlive.timeout <= Duration.zero) { + throw ArgumentError.value(keepAlive.timeout, 'keepAlive.timeout', 'must be positive'); + } +} + +/// `scheme://host:port` of [url]: the calls that share a connection. +Uri originOf(Uri url) => Uri(scheme: url.scheme, host: url.host, port: url.port); + +/// Runs one client's [NativeKeepAlive]: a loop per origin that has calls open, which ends once none are. +final class KeepAliveMonitor { + KeepAliveMonitor(this.config, {required this.ping}); + + final NativeKeepAlive config; + + /// Sends one keepalive request to [origin], and completes once it has ended, however it ended. + final Future Function(Uri origin) ping; + + final _origins = {}; + bool _closed = false; + + /// Keeps the connection to [url]'s origin busy until the returned function is called, which the caller does once + /// the call has finished. + void Function() track(Uri url) { + final origin = originOf(url); + final state = _origins.putIfAbsent(origin, _Origin.new); + final call = Object(); + state.calls.add(call); + if (!state.running && !_closed) { + state.running = true; + unawaited(_run(origin, state)); + } + return () => state.calls.remove(call); + } + + /// Stops sending. A keepalive in flight is left to finish. + void close() => _closed = true; + + Future _run(Uri origin, _Origin state) async { + try { + while (true) { + await Future.delayed(config.interval); + if (_closed || state.calls.isEmpty) { + return; + } + await ping(origin); + } + } finally { + state.running = false; + if (state.calls.isEmpty && identical(_origins[origin], state)) { + _origins.remove(origin); + } + } + } +} + +final class _Origin { + /// One token per open call. + final calls = {}; + bool running = false; +} diff --git a/lib/src/cronet/cronet_http_client.dart b/lib/src/cronet/cronet_http_client.dart index a00e89a..1d782bf 100644 --- a/lib/src/cronet/cronet_http_client.dart +++ b/lib/src/cronet/cronet_http_client.dart @@ -1,6 +1,7 @@ import 'package:flutter/services.dart'; import '../channel/channel_http_client.dart'; +import '../channel/keep_alive.dart'; import '../native_http_config.dart'; import '../native_http_exception.dart'; import 'cronet_error.dart'; @@ -32,6 +33,7 @@ final class CronetHttpClient extends ChannelHttpClient { CronetHttpClient._(int engineId, NativeHttpConfig config) : super(cronetBridge, engineId, config); static Future create(NativeHttpConfig config) async { + validateKeepAlive(config.keepAlive); final idle = config.quicIdleTimeout; if (idle != null && idle < const Duration(seconds: 1)) { throw ArgumentError.value(idle, 'quicIdleTimeout', 'Cronet takes whole seconds; use at least 1 s'); diff --git a/lib/src/native_http_config.dart b/lib/src/native_http_config.dart index 1a145f8..90014bf 100644 --- a/lib/src/native_http_config.dart +++ b/lib/src/native_http_config.dart @@ -9,10 +9,38 @@ final class NativeHttpResponseInfo { final String? protocol; } +/// Application-level keepalive: while a client has calls open to an origin, it sends a `grpc.health.v1.Health/Check` +/// request (Connect protocol, an empty `HealthCheckRequest`) every [interval] over the same platform stack, so the +/// connection carries traffic even while its calls are quiet. Register that service on every server the client +/// calls; in connect-go that is `grpchealth.Register`. +/// +/// URLSession has no way to send HTTP/2 PING frames, and this plugin doesn't turn on Cronet's (an experimental +/// option), so a quiet HTTP/2 connection is otherwise silent, and anything on the path that closes idle connections +/// can close it: a load balancer's idle timeout (an AWS ALB's is 60 s by default), a NAT mapping, or the server's own. +/// A load balancer that ends HTTP/2 also answers PINGs itself, while these requests reach the server. Over HTTP/3 the +/// server's QUIC pings already keep the path busy. +/// +/// It only keeps the connection busy: it doesn't fail calls or notice a dead connection, and a request on another +/// stream doesn't count as data on a call's own stream. Whatever the outcome, the next request waits [interval]. One +/// that hasn't ended within [timeout] is canceled. A response other than HTTP 200 is reported once per origin +/// through `FlutterError.onError`, since it usually means the server doesn't serve the health service. +/// +/// Each request is small, but on a phone a short interval keeps the radio awake while calls are open. +final class NativeKeepAlive { + const NativeKeepAlive({this.interval = const Duration(seconds: 15), this.timeout = const Duration(seconds: 10)}); + + /// The wait before each request, starting when a call opens to an origin with none open, and after each request + /// ends. Keep it well under the shortest idle timeout on the path. + final Duration interval; + + /// How long a request may take before it is canceled. + final Duration timeout; +} + /// Configuration shared by both platform clients. Server certificates are checked against the platform's trust /// store. final class NativeHttpConfig { - const NativeHttpConfig({this.http3Origins = const {}, this.quicIdleTimeout, this.onResponse}); + const NativeHttpConfig({this.http3Origins = const {}, this.quicIdleTimeout, this.onResponse, this.keepAlive}); /// `https://host:port` origins to try HTTP/3 with from the first request, before an `Alt-Svc` response header has /// advertised it. Both platforms fall back to HTTP/2 over TCP on their own when QUIC fails. @@ -32,4 +60,7 @@ final class NativeHttpConfig { /// Called with each response's headers, before its body is read. An exception it throws goes to /// `FlutterError.onError`, and the call goes on. final void Function(NativeHttpResponseInfo info)? onResponse; + + /// Application-level keepalive requests while this client has calls open. `null`, the default, sends none. + final NativeKeepAlive? keepAlive; } diff --git a/lib/src/url_session/url_session_http_client.dart b/lib/src/url_session/url_session_http_client.dart index f158e93..4fe6402 100644 --- a/lib/src/url_session/url_session_http_client.dart +++ b/lib/src/url_session/url_session_http_client.dart @@ -2,6 +2,7 @@ import 'package:connectrpc/connect.dart'; import 'package:flutter/services.dart'; import '../channel/channel_http_client.dart'; +import '../channel/keep_alive.dart'; import '../native_http_config.dart'; import '../native_http_exception.dart'; import 'ns_url_error.dart'; @@ -40,6 +41,7 @@ final class UrlSessionHttpClient extends ChannelHttpClient { UrlSessionHttpClient._(int engineId, NativeHttpConfig config) : super(urlSessionBridge, engineId, config); static Future create(NativeHttpConfig config) async { + validateKeepAlive(config.keepAlive); final engineId = await ChannelHttpClient.createEngine(urlSessionBridge, { 'http3Origins': [ for (final origin in config.http3Origins) [origin.host, origin.port], diff --git a/test/cronet_http_client_test.dart b/test/cronet_http_client_test.dart index 241f538..1b77ffb 100644 --- a/test/cronet_http_client_test.dart +++ b/test/cronet_http_client_test.dart @@ -7,70 +7,7 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; -/// Stands in for ConnectNativeHttpPlugin.kt: records method calls and lets a test emit the plugin's events. -class FakePlugin { - FakePlugin() { - _messenger.setMockMethodCallHandler(cronetMethodChannel, (call) async { - calls.add(call); - if (refusing.contains(call.method)) { - throw PlatformException(code: 'badArguments', message: '${call.method} refused'); - } - return call.method == 'createEngine' ? 7 : null; - }); - _messenger.setMockMethodCallHandler(MethodChannel(cronetEventChannel.name), (call) async => null); - } - - final calls = []; - - /// Methods the plugin answers with an error. - final refusing = {}; - - TestDefaultBinaryMessenger get _messenger => TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; - - List callsTo(String method) => calls.where((c) => c.method == method).toList(); - - int get lastCallId => callsTo('open').last.arguments['callId'] as int; - - Future emit(Map event) async { - await _messenger.handlePlatformMessage( - cronetEventChannel.name, - const StandardMethodCodec().encodeSuccessEnvelope(event), - (_) {}, - ); - } - - Future headers(int callId, {String protocol = 'quic/1+spdy/3'}) => emit({ - 'callId': callId, - 'type': 'headers', - 'status': 200, - 'protocol': protocol, - 'headers': [ - ['content-type', 'application/connect+proto'], - ], - }); - - Future data(int callId, List bytes) => - emit({'callId': callId, 'type': 'data', 'data': Uint8List.fromList(bytes)}); - - Future failed(int callId, {required bool streamReady, int? internalErrorCode}) => emit({ - 'callId': callId, - 'type': 'failed', - 'streamReady': streamReady, - 'message': 'net::ERR', - 'internalErrorCode': internalErrorCode, - }); -} - -/// Lets queued microtasks and platform-channel replies run. -Future settle() => Future.delayed(Duration.zero); - -HttpRequest request(Stream? body, {AbortSignal? signal}) => HttpRequest( - 'https://example.test/svc/Method', - 'POST', - Headers()..add('content-type', 'application/proto'), - body, - signal, -); +import 'fake_cronet_plugin.dart'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); diff --git a/test/fake_cronet_plugin.dart b/test/fake_cronet_plugin.dart new file mode 100644 index 0000000..9a31c00 --- /dev/null +++ b/test/fake_cronet_plugin.dart @@ -0,0 +1,70 @@ +import 'package:connect_native_http/src/cronet/cronet_http_client.dart'; +import 'package:connectrpc/connect.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Stands in for ConnectNativeHttpPlugin.kt: records method calls and lets a test emit the plugin's events. +class FakePlugin { + FakePlugin() { + _messenger.setMockMethodCallHandler(cronetMethodChannel, (call) async { + calls.add(call); + if (refusing.contains(call.method)) { + throw PlatformException(code: 'badArguments', message: '${call.method} refused'); + } + return call.method == 'createEngine' ? 7 : null; + }); + _messenger.setMockMethodCallHandler(MethodChannel(cronetEventChannel.name), (call) async => null); + } + + final calls = []; + + /// Methods the plugin answers with an error. + final refusing = {}; + + TestDefaultBinaryMessenger get _messenger => TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + List callsTo(String method) => calls.where((c) => c.method == method).toList(); + + int get lastCallId => callsTo('open').last.arguments['callId'] as int; + + Future emit(Map event) async { + await _messenger.handlePlatformMessage( + cronetEventChannel.name, + const StandardMethodCodec().encodeSuccessEnvelope(event), + (_) {}, + ); + } + + Future headers(int callId, {String protocol = 'quic/1+spdy/3'}) => emit({ + 'callId': callId, + 'type': 'headers', + 'status': 200, + 'protocol': protocol, + 'headers': [ + ['content-type', 'application/connect+proto'], + ], + }); + + Future data(int callId, List bytes) => + emit({'callId': callId, 'type': 'data', 'data': Uint8List.fromList(bytes)}); + + Future failed(int callId, {required bool streamReady, int? internalErrorCode}) => emit({ + 'callId': callId, + 'type': 'failed', + 'streamReady': streamReady, + 'message': 'net::ERR', + 'internalErrorCode': internalErrorCode, + }); +} + +/// Lets queued microtasks and platform-channel replies run. +Future settle() => Future.delayed(Duration.zero); + +HttpRequest request(Stream? body, {AbortSignal? signal, String url = 'https://example.test/svc/Method'}) => + HttpRequest( + url, + 'POST', + Headers()..add('content-type', 'application/proto'), + body, + signal, + ); diff --git a/test/keep_alive_test.dart b/test/keep_alive_test.dart new file mode 100644 index 0000000..0b4869a --- /dev/null +++ b/test/keep_alive_test.dart @@ -0,0 +1,202 @@ +import 'dart:async'; + +import 'package:connect_native_http/connect_native_http.dart'; +import 'package:connect_native_http/src/cronet/cronet_http_client.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'fake_cronet_plugin.dart'; + +const interval = Duration(milliseconds: 30); +const timeout = Duration(milliseconds: 60); +const keepAlive = NativeKeepAlive(interval: interval, timeout: timeout); +const healthCheck = 'https://example.test/grpc.health.v1.Health/Check'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late FakePlugin plugin; + final clients = []; + + setUp(() => plugin = FakePlugin()); + + // A client's keepalive loop would otherwise go on into the next test, against that test's fake plugin. + tearDown(() async { + for (final client in clients) { + await client.close(); + } + clients.clear(); + }); + + Future create(NativeHttpConfig config) async { + final client = await CronetHttpClient.create(config); + clients.add(client); + return client; + } + + /// The `open` arguments of each request to [url] so far. + List> opensTo(String url) => [ + for (final call in plugin.callsTo('open')) + if ((call.arguments as Map)['url'] == url) call.arguments as Map, + ]; + + List callIds(List> opens) => [for (final open in opens) open['callId']! as int]; + + Future until(String what, bool Function() condition) async { + final deadline = DateTime.now().add(const Duration(seconds: 2)); + while (!condition()) { + if (DateTime.now().isAfter(deadline)) { + fail('timed out waiting until $what; calls: ${plugin.calls}'); + } + await Future.delayed(const Duration(milliseconds: 5)); + } + } + + /// Answers call [id] with [status] and ends it. + Future answer(int id, {int status = 200}) async { + await plugin.emit({ + 'callId': id, + 'type': 'headers', + 'status': status, + 'protocol': 'h2', + 'headers': [ + ['content-type', 'application/proto'], + ], + }); + await plugin.emit({'callId': id, 'type': 'done'}); + } + + /// Opens a call to [url] whose request side stays open, and returns its id. The call must not fail: an error on + /// its response would surface as an uncaught error and fail the test. + Future openCall(CronetHttpClient client, {String url = 'https://example.test/svc/Method'}) async { + unawaited(client.httpClient(request(StreamController().stream, url: url))); + await settle(); + return callIds(opensTo(url)).last; + } + + test('sends no keepalives unless configured', () async { + final client = await create(const NativeHttpConfig()); + await openCall(client); + await Future.delayed(interval * 4); + expect(opensTo(healthCheck), isEmpty); + }); + + test('sends a Connect health check to the origin of an open call, after the interval', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + expect(opensTo(healthCheck), isEmpty, reason: 'nothing before the first interval'); + await until('a keepalive was sent', () => opensTo(healthCheck).isNotEmpty); + final open = opensTo(healthCheck).single; + expect(open['engineId'], 7); + expect(open['method'], 'POST'); + expect(open['hasBody'], isFalse, reason: 'an empty HealthCheckRequest encodes to no bytes'); + expect(open['headers'], [ + ['content-type', 'application/proto'], + ['connect-protocol-version', '1'], + ['connect-timeout-ms', '60'], + ]); + }); + + test('keeps sending while a call is open, and stops once none is', () async { + final responses = []; + final client = await create(NativeHttpConfig(keepAlive: keepAlive, onResponse: responses.add)); + final call = await openCall(client); + for (var n = 1; n <= 3; n++) { + await until('keepalive $n was sent', () => opensTo(healthCheck).length == n); + await answer(callIds(opensTo(healthCheck)).last); + } + expect(responses, isEmpty, reason: 'onResponse sees the app\'s calls only'); + + // The call ends. A keepalive already waiting for its interval may still go out, but nothing after it. + await answer(call); + await Future.delayed(interval * 2); + for (final id in callIds(opensTo(healthCheck)).skip(3)) { + await answer(id); + } + final sent = opensTo(healthCheck).length; + await Future.delayed(interval * 4); + expect(opensTo(healthCheck), hasLength(sent)); + expect(responses, hasLength(1)); + }); + + test('a keepalive with no answer is canceled after the timeout, the call carries on, and the next follows', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + final call = await openCall(client); + await until('a keepalive was sent', () => opensTo(healthCheck).isNotEmpty); + final first = callIds(opensTo(healthCheck)).single; + + await until('the keepalive was canceled', () => plugin.callsTo('cancel').isNotEmpty); + expect(plugin.callsTo('cancel').single.arguments['callId'], first); + await plugin.emit({'callId': first, 'type': 'canceled'}); + + await until('the next keepalive was sent', () => opensTo(healthCheck).length == 2); + expect(plugin.callsTo('cancel').map((c) => c.arguments['callId']), isNot(contains(call))); + }); + + test('a failed keepalive fails no call, and the next follows', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + final call = await openCall(client); + await until('a keepalive was sent', () => opensTo(healthCheck).isNotEmpty); + await plugin.failed(callIds(opensTo(healthCheck)).single, streamReady: true, internalErrorCode: -101); + await until('the next keepalive was sent', () => opensTo(healthCheck).length == 2); + expect(plugin.callsTo('cancel').map((c) => c.arguments['callId']), isNot(contains(call))); + }); + + test('one keepalive at a time per origin, however many calls are open to it', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + await openCall(client, url: 'https://example.test/svc/Other'); + await openCall(client, url: 'https://other.test:8443/svc/Method'); + await until('each origin got a keepalive', () { + return opensTo(healthCheck).isNotEmpty && + opensTo('https://other.test:8443/grpc.health.v1.Health/Check').isNotEmpty; + }); + // The first is still waiting for an answer, so no second one has gone out to the same origin. + await Future.delayed(interval * 2 ~/ 3); + expect(opensTo(healthCheck), hasLength(1)); + }); + + test('close stops keepalives', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + await client.close(); + await Future.delayed(interval * 4); + expect(opensTo(healthCheck), isEmpty); + }); + + test('rejects intervals and timeouts that are not positive', () async { + await expectLater( + CronetHttpClient.create(const NativeHttpConfig(keepAlive: NativeKeepAlive(interval: Duration.zero))), + throwsArgumentError, + ); + await expectLater( + CronetHttpClient.create(const NativeHttpConfig(keepAlive: NativeKeepAlive(timeout: Duration(seconds: -1)))), + throwsArgumentError, + ); + expect(plugin.callsTo('createEngine'), isEmpty); + }); + + group('reporting', () { + final reported = []; + late FlutterExceptionHandler? previousOnError; + + setUp(() { + reported.clear(); + previousOnError = FlutterError.onError; + FlutterError.onError = reported.add; + }); + + tearDown(() => FlutterError.onError = previousOnError); + + test('a response other than HTTP 200 is reported once per origin, and keepalives go on', () async { + final client = await create(const NativeHttpConfig(keepAlive: keepAlive)); + await openCall(client); + for (var n = 1; n <= 2; n++) { + await until('keepalive $n was sent', () => opensTo(healthCheck).length == n); + await answer(callIds(opensTo(healthCheck)).last, status: 404); + } + await until('the next keepalive was sent', () => opensTo(healthCheck).length == 3); + expect(reported, hasLength(1)); + expect('${reported.single.exception}', allOf(contains('HTTP 404'), contains('grpc.health.v1.Health'))); + }); + }); +} diff --git a/test_server/freeze.go b/test_server/freeze.go index 06704ac..035f3e5 100644 --- a/test_server/freeze.go +++ b/test_server/freeze.go @@ -63,14 +63,17 @@ func (c *freezablePacketConn) SetWriteBuffer(bytes int) error { return errors.New("the underlying conn can't set its send buffer") } -// adminHandler lets tests on a device freeze the HTTP/3 path over plain HTTP: +// adminHandler lets tests on a device freeze the HTTP/3 path, and see the health checks the server received, over +// plain HTTP: // // POST /freeze?for=10s drop all UDP traffic for 10 s // POST /unfreeze stop dropping now +// GET /health-checks every health check so far, as JSON (see healthLog) // // TCP is never frozen, so HTTP/2 calls are unaffected. -func adminHandler(conn *freezablePacketConn) http.Handler { +func adminHandler(conn *freezablePacketConn, health *healthLog) http.Handler { mux := http.NewServeMux() + mux.HandleFunc("GET /health-checks", health.serveJSON) mux.HandleFunc("POST /freeze", func(w http.ResponseWriter, r *http.Request) { d, err := time.ParseDuration(r.URL.Query().Get("for")) if err != nil || d <= 0 { diff --git a/test_server/go.mod b/test_server/go.mod index 7956202..431a203 100644 --- a/test_server/go.mod +++ b/test_server/go.mod @@ -4,6 +4,7 @@ go 1.26.2 require ( connectrpc.com/connect/v2 v2.0.1 + connectrpc.com/grpchealth/v2 v2.0.0 github.com/quic-go/quic-go v0.65.0 google.golang.org/protobuf v1.36.12 ) diff --git a/test_server/go.sum b/test_server/go.sum index 6e8adb6..bcbae76 100644 --- a/test_server/go.sum +++ b/test_server/go.sum @@ -1,5 +1,7 @@ connectrpc.com/connect/v2 v2.0.1 h1:WTAycxe87UqxSM5oxS30d1dalZ+C5xz8Ec8O6zVYkHQ= connectrpc.com/connect/v2 v2.0.1/go.mod h1:N32OLrstcPUzkykFynnefFro3+xzSdOrZlK2jHmS/xo= +connectrpc.com/grpchealth/v2 v2.0.0 h1:GOBfwE25Q70Kt0QhPawK9BKsh2tP8X0EUGeIk5Sz/ag= +connectrpc.com/grpchealth/v2 v2.0.0/go.mod h1:85+eI11LlzfuyFXX/IuB/xzEfRVdJM3/RtoMgV9AEV4= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= diff --git a/test_server/health.go b/test_server/health.go new file mode 100644 index 0000000..0b19931 --- /dev/null +++ b/test_server/health.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "net/http" + "sync" + + "connectrpc.com/grpchealth/v2" +) + +// healthCheckPath is the procedure connect_native_http's keepalive calls. +const healthCheckPath = "/" + grpchealth.HealthV1ServiceName + "/Check" + +// healthLog records each health check the server receives: its protocol, and the client's address, which identifies +// the connection it came on. The integration tests read it through the admin endpoint to see that a client's +// keepalives arrived on the same connection as the call they keep busy. +type healthLog struct { + mu sync.Mutex + entries []healthLogEntry +} + +type healthLogEntry struct { + Proto string `json:"proto"` + RemoteAddr string `json:"remoteAddr"` +} + +// record wraps next, logging each request for healthCheckPath before passing it on. +func (l *healthLog) record(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == healthCheckPath { + l.mu.Lock() + l.entries = append(l.entries, healthLogEntry{Proto: r.Proto, RemoteAddr: r.RemoteAddr}) + l.mu.Unlock() + } + next.ServeHTTP(w, r) + }) +} + +// serveJSON answers with every check logged so far, oldest first. +func (l *healthLog) serveJSON(w http.ResponseWriter, _ *http.Request) { + l.mu.Lock() + entries := append([]healthLogEntry{}, l.entries...) + l.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(entries) +} diff --git a/test_server/main.go b/test_server/main.go index 10db368..b33bccf 100644 --- a/test_server/main.go +++ b/test_server/main.go @@ -8,7 +8,8 @@ // // -max-idle and -keepalive set the server's QUIC idle timeout and ping interval. QUIC uses the smaller of the two // endpoints' idle timeouts, so -max-idle also bounds how long a client takes to notice a dead path. -admin-addr -// serves a plain-HTTP endpoint that freezes the HTTP/3 path (see adminHandler). +// serves a plain-HTTP endpoint that freezes the HTTP/3 path and lists the health checks received (see +// adminHandler). package main import ( @@ -29,6 +30,7 @@ import ( "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" + "connectrpc.com/grpchealth/v2" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" "github.com/quic-go/quic-go/qlog" @@ -66,7 +68,8 @@ func main() { log.Fatal(err) } - handler := protocolHeaders(newElizaMux(), *altSvc) + health := &healthLog{} + handler := protocolHeaders(health.record(newElizaMux()), *altSvc) tlsConfig := newTLSConfig(cert) h2 := &http.Server{Addr: *addr, Handler: handler, TLSConfig: tlsConfig, ReadHeaderTimeout: 10 * time.Second} go func() { @@ -89,7 +92,7 @@ func main() { } }() if *adminAddr != "" { - admin := &http.Server{Addr: *adminAddr, Handler: adminHandler(conn), ReadHeaderTimeout: 10 * time.Second} + admin := &http.Server{Addr: *adminAddr, Handler: adminHandler(conn, health), ReadHeaderTimeout: 10 * time.Second} go func() { if err := admin.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("admin: %v", err) @@ -125,9 +128,11 @@ func newHTTP3Server(handler http.Handler, tlsConfig *tls.Config, quicConfig *qui } } +// newElizaMux serves the eliza service, and gRPC's health service for connect_native_http's keepalive. func newElizaMux() *http.ServeMux { server := connect.NewServer() elizav1connect.RegisterElizaServiceHandler(server, elizaServer{}) + grpchealth.Register(server, grpchealth.NewStaticChecker()) mux := http.NewServeMux() connecthttp.Mount(mux, server) // A 307 makes a client send the same request body again, to the path after /redirect. The body is read first: @@ -142,11 +147,13 @@ func newElizaMux() *http.ServeMux { } // protocolHeaders logs each request and reports the protocol it arrived on in X-Proto, which is how tests check -// for HTTP/3 on clients that don't expose the negotiated protocol. HTTP/2 responses also advertise HTTP/3. +// for HTTP/3 on clients that don't expose the negotiated protocol, and the client's address in X-Remote-Addr, which +// identifies the connection. HTTP/2 responses also advertise HTTP/3. func protocolHeaders(next http.Handler, altSvc string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { log.Printf("%s %s %s from %s", r.Proto, r.Method, r.URL.Path, r.RemoteAddr) w.Header().Set("X-Proto", r.Proto) + w.Header().Set("X-Remote-Addr", r.RemoteAddr) if r.ProtoMajor < 3 && altSvc != "" { w.Header().Set("Alt-Svc", altSvc) } diff --git a/test_server/server_test.go b/test_server/server_test.go index 756b383..6466477 100644 --- a/test_server/server_test.go +++ b/test_server/server_test.go @@ -1,9 +1,11 @@ package main import ( + "bytes" "context" "crypto/tls" "crypto/x509" + "encoding/json" "errors" "io" "net" @@ -11,6 +13,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "sync" "testing" "time" @@ -168,7 +171,7 @@ func TestFrozenPathFailsOpenStreamWithinIdleTimeout(t *testing.T) { func TestAdminFreezeAndUnfreeze(t *testing.T) { conn := &freezablePacketConn{} - handler := adminHandler(conn) + handler := adminHandler(conn, &healthLog{}) post := func(target string) int { w := httptest.NewRecorder() handler.ServeHTTP(w, httptest.NewRequest(http.MethodPost, target, nil)) @@ -381,3 +384,44 @@ func TestAltSvcOnlyOnTCPResponses(t *testing.T) { } } } + +func TestHealthCheckIsServedAndLogged(t *testing.T) { + health := &healthLog{} + handler := protocolHeaders(health.record(newElizaMux()), "") + post := func(path string) *httptest.ResponseRecorder { + // An empty body: a HealthCheckRequest for the whole server, or an empty SayRequest. + req := httptest.NewRequest(http.MethodPost, path, nil) + req.Header.Set("Content-Type", "application/proto") + req.Header.Set("Connect-Protocol-Version", "1") + req.RemoteAddr = "192.0.2.1:1234" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w + } + + w := post(healthCheckPath) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body) + } + // A HealthCheckResponse with status SERVING: field 1, varint 1. + if got, want := w.Body.Bytes(), []byte{0x08, 0x01}; !bytes.Equal(got, want) { + t.Errorf("body = %x, want %x", got, want) + } + if got := w.Header().Get("X-Remote-Addr"); got != "192.0.2.1:1234" { + t.Errorf("X-Remote-Addr = %q", got) + } + if code := post("/connectrpc.eliza.v1.ElizaService/Say").Code; code != http.StatusOK { + t.Fatalf("Say: status %d", code) + } + + admin := httptest.NewRecorder() + adminHandler(&freezablePacketConn{}, health).ServeHTTP(admin, httptest.NewRequest(http.MethodGet, "/health-checks", nil)) + var entries []healthLogEntry + if err := json.Unmarshal(admin.Body.Bytes(), &entries); err != nil { + t.Fatalf("%v: %s", err, admin.Body) + } + want := []healthLogEntry{{Proto: "HTTP/1.1", RemoteAddr: "192.0.2.1:1234"}} + if !slices.Equal(entries, want) { + t.Errorf("health checks = %+v, want %+v (the Say call isn't one)", entries, want) + } +}