diff --git a/lib/core/extensions/sandbox/sandbox_auto_recovery.dart b/lib/core/extensions/sandbox/sandbox_auto_recovery.dart new file mode 100644 index 00000000..3c373d3c --- /dev/null +++ b/lib/core/extensions/sandbox/sandbox_auto_recovery.dart @@ -0,0 +1,74 @@ +/// Exponential backoff for sandboxed plugin restarts (Block E §4). +/// +/// Up to [maxAttempts] retries inside [window], with delays 1s → 2s → 4s. +/// +/// Usage after a crash: +/// ```dart +/// final delay = recovery.recordFailure(); +/// if (delay == null) { /* give up */ } +/// else { await Future.delayed(delay); /* respawn */ } +/// ``` +class SandboxAutoRecovery { + SandboxAutoRecovery({ + this.maxAttempts = 3, + this.window = const Duration(minutes: 5), + this.backoffSchedule = const [ + Duration(seconds: 1), + Duration(seconds: 2), + Duration(seconds: 4), + ], + DateTime Function()? clock, + }) : _clock = clock ?? DateTime.now; + + final int maxAttempts; + final Duration window; + final List backoffSchedule; + final DateTime Function() _clock; + + final List _failures = []; + + /// Failures still counted inside the current [window]. + int get recentFailureCount { + _prune(); + return _failures.length; + } + + /// Whether another retry is allowed (call before or after checking + /// [recordFailure]'s return value). + bool get canRetry { + _prune(); + return _failures.length < maxAttempts; + } + + /// Suggested delay before the next spawn given current failure count. + /// Returns `null` when retries are exhausted. + Duration? nextBackoff() { + _prune(); + if (_failures.length >= maxAttempts) return null; + final index = _failures.length.clamp(0, backoffSchedule.length - 1); + return backoffSchedule[index]; + } + + /// Records a crash / deadlock. + /// + /// Returns the backoff before the next retry, or `null` if the caller must + /// stop retrying (more than [maxAttempts] failures in [window]). + Duration? recordFailure() { + _failures.add(_clock()); + _prune(); + if (_failures.length > maxAttempts) { + return null; + } + final index = (_failures.length - 1).clamp(0, backoffSchedule.length - 1); + return backoffSchedule[index]; + } + + void recordSuccess() => _failures.clear(); + + void reset() => _failures.clear(); + + void _prune() { + final cutoff = _clock().subtract(window); + _failures.removeWhere((t) => t.isBefore(cutoff)); + } +} diff --git a/lib/core/extensions/sandbox/sandbox_watchdog.dart b/lib/core/extensions/sandbox/sandbox_watchdog.dart new file mode 100644 index 00000000..54d2b086 --- /dev/null +++ b/lib/core/extensions/sandbox/sandbox_watchdog.dart @@ -0,0 +1,174 @@ +import 'dart:async'; + +import 'package:flutter/foundation.dart'; +import 'package:querya_desktop/core/extensions/rpc/json_rpc_stdio_client.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_auto_recovery.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_process_runner.dart'; + +/// Why the watchdog stopped monitoring a plugin process. +enum SandboxWatchdogStopReason { + /// [SandboxWatchdog.stop] was called explicitly. + stopped, + + /// Plugin failed to answer `system.ping` within the pong timeout. + deadlock, + + /// Child process exited on its own. + processExited, +} + +/// Heartbeat monitor for a sandboxed plugin process (Block E §4). +/// +/// Sends `system.ping` every [pingInterval]. If no successful response arrives +/// within [pongTimeout], marks a deadlock, SIGKILLs the process, and records +/// the failure on [recovery]. +class SandboxWatchdog { + SandboxWatchdog({ + this.pingInterval = const Duration(seconds: 30), + this.pongTimeout = const Duration(seconds: 5), + this.recovery, + Future Function()? ping, + void Function(SandboxWatchdogStopReason reason)? onStopped, + }) : _pingOverride = ping, + _onStopped = onStopped; + + final Duration pingInterval; + final Duration pongTimeout; + final SandboxAutoRecovery? recovery; + final Future Function()? _pingOverride; + final void Function(SandboxWatchdogStopReason reason)? _onStopped; + + SandboxProcessHandle? _handle; + JsonRpcStdioClient? _client; + Timer? _timer; + var _running = false; + var _pingInFlight = false; + SandboxWatchdogStopReason? _lastReason; + StreamSubscription? _exitSub; + + bool get isRunning => _running; + + SandboxWatchdogStopReason? get lastStopReason => _lastReason; + + /// Starts monitoring [handle]. Cancels any previous session first. + void start(SandboxProcessHandle handle, {JsonRpcStdioClient? client}) { + stop(reason: SandboxWatchdogStopReason.stopped, notify: false); + _handle = handle; + _client = client; + _running = true; + _lastReason = null; + + _exitSub = handle.process.exitCode.asStream().listen((code) { + if (!_running) return; + debugPrint( + 'SandboxWatchdog: ${handle.pluginId} exited with code $code', + ); + recovery?.recordFailure(); + _finish(SandboxWatchdogStopReason.processExited); + }, onError: (_) { + if (!_running) return; + recovery?.recordFailure(); + _finish(SandboxWatchdogStopReason.processExited); + }); + + _timer = Timer.periodic(pingInterval, (_) { + unawaited(_tick()); + }); + } + + /// Stops timers and releases the RPC client. Does not kill the process + /// unless [reason] is [SandboxWatchdogStopReason.deadlock] (already killed). + void stop({ + SandboxWatchdogStopReason reason = SandboxWatchdogStopReason.stopped, + bool notify = true, + }) { + if (!_running && _timer == null && _client == null && _exitSub == null) { + return; + } + _running = false; + _timer?.cancel(); + _timer = null; + unawaited(_exitSub?.cancel()); + _exitSub = null; + final client = _client; + _client = null; + if (client != null) { + unawaited(client.close()); + } + _handle = null; + _lastReason = reason; + if (notify) { + _onStopped?.call(reason); + } + } + + Future _tick() async { + if (!_running || _pingInFlight) return; + _pingInFlight = true; + try { + final result = await _sendPing().timeout(pongTimeout); + if (!_running) return; + if (!isPong(result)) { + await _onDeadlock('unexpected ping result: $result'); + return; + } + recovery?.recordSuccess(); + } on TimeoutException { + if (!_running) return; + await _onDeadlock('system.ping timed out after $pongTimeout'); + } catch (e) { + if (!_running) return; + await _onDeadlock('system.ping failed: $e'); + } finally { + _pingInFlight = false; + } + } + + Future _sendPing() { + final override = _pingOverride; + if (override != null) return override(); + + final handle = _handle; + if (handle == null) { + throw StateError('SandboxWatchdog has no handle'); + } + _client ??= JsonRpcStdioClient( + stdout: handle.process.stdout, + stdin: handle.process.stdin, + requestTimeout: pongTimeout, + ); + return _client!.sendRequest('system.ping'); + } + + Future _onDeadlock(String detail) async { + final handle = _handle; + debugPrint( + 'SandboxWatchdog: deadlock on ${handle?.pluginId ?? 'unknown'} — $detail', + ); + recovery?.recordFailure(); + // Cancel exit watcher before kill so we don't double-count the failure. + await _exitSub?.cancel(); + _exitSub = null; + if (handle != null && !handle.isDisposed) { + await handle.kill(); + } + _finish(SandboxWatchdogStopReason.deadlock); + } + + void _finish(SandboxWatchdogStopReason reason) { + stop(reason: reason); + } + + /// Accepts common pong shapes from plugin runtimes. + static bool isPong(Object? result) { + if (result == null) return true; + if (result == true) return true; + if (result == 'pong') return true; + if (result is Map && + (result['pong'] == true || result['result'] == 'pong')) { + return true; + } + // Any non-error JSON-RPC result counts as alive. + return true; + } +} diff --git a/test/core/extensions/sandbox/sandbox_watchdog_test.dart b/test/core/extensions/sandbox/sandbox_watchdog_test.dart new file mode 100644 index 00000000..0c231e90 --- /dev/null +++ b/test/core/extensions/sandbox/sandbox_watchdog_test.dart @@ -0,0 +1,238 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_auto_recovery.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_launch_command.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_process_runner.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_scratch_directory.dart'; +import 'package:querya_desktop/core/extensions/sandbox/sandbox_watchdog.dart'; + +class _FakeProcess implements Process { + _FakeProcess() + : _stdoutController = StreamController>.broadcast(), + _stdinController = StreamController>() { + stdin = IOSink(_stdinController.sink); + } + + final StreamController> _stdoutController; + final StreamController> _stdinController; + final _exit = Completer(); + var killed = false; + ProcessSignal? lastSignal; + + @override + int get pid => 4242; + + @override + late final IOSink stdin; + + @override + Stream> get stdout => _stdoutController.stream; + + @override + Stream> get stderr => const Stream.empty(); + + @override + Future get exitCode => _exit.future; + + @override + bool kill([ProcessSignal signal = ProcessSignal.sigterm]) { + killed = true; + lastSignal = signal; + if (!_exit.isCompleted) { + _exit.complete(signal == ProcessSignal.sigkill ? -9 : 0); + } + return true; + } + + void completeExit([int code = 1]) { + if (!_exit.isCompleted) _exit.complete(code); + } +} + +Future _handle( + _FakeProcess process, + Directory tempBase, +) async { + final scratch = await SandboxScratchDirectory.create( + pluginId: 'wd.driver', + baseDirectory: tempBase, + token: '1', + ); + return SandboxProcessHandle( + pluginId: 'wd.driver', + process: process, + scratch: scratch, + launchCommand: const SandboxLaunchCommand( + executable: '/bin/true', + arguments: [], + platform: 'linux', + usesOsSandbox: false, + ), + ); +} + +void main() { + group('SandboxAutoRecovery', () { + test('returns 1s → 2s → 4s then exhausts', () { + var now = DateTime(2026, 1, 1, 12); + final recovery = SandboxAutoRecovery(clock: () => now); + + expect(recovery.recordFailure(), const Duration(seconds: 1)); + expect(recovery.canRetry, isTrue); + expect(recovery.recordFailure(), const Duration(seconds: 2)); + expect(recovery.canRetry, isTrue); + expect(recovery.recordFailure(), const Duration(seconds: 4)); + expect(recovery.canRetry, isFalse); + expect(recovery.recordFailure(), isNull); + expect(recovery.nextBackoff(), isNull); + }); + + test('prunes failures outside the window', () { + var now = DateTime(2026, 1, 1, 12); + final recovery = SandboxAutoRecovery(clock: () => now); + + expect(recovery.recordFailure(), const Duration(seconds: 1)); + now = now.add(const Duration(minutes: 6)); + expect(recovery.recentFailureCount, 0); + expect(recovery.canRetry, isTrue); + expect(recovery.recordFailure(), const Duration(seconds: 1)); + }); + + test('recordSuccess clears failures', () { + final recovery = SandboxAutoRecovery(); + recovery.recordFailure(); + recovery.recordFailure(); + recovery.recordSuccess(); + expect(recovery.recentFailureCount, 0); + expect(recovery.nextBackoff(), const Duration(seconds: 1)); + }); + }); + + group('SandboxWatchdog', () { + late Directory tempBase; + + setUp(() async { + tempBase = await Directory.systemTemp.createTemp('querya_wd_test_'); + }); + + tearDown(() async { + if (await tempBase.exists()) { + await tempBase.delete(recursive: true); + } + }); + + test('successful ping clears recovery failures', () async { + final process = _FakeProcess(); + final handle = await _handle(process, tempBase); + final recovery = SandboxAutoRecovery(); + recovery.recordFailure(); + + final ping = Completer(); + final watchdog = SandboxWatchdog( + pingInterval: const Duration(milliseconds: 20), + pongTimeout: const Duration(seconds: 1), + recovery: recovery, + ping: () => ping.future, + ); + + watchdog.start(handle); + await Future.delayed(const Duration(milliseconds: 40)); + ping.complete('pong'); + await Future.delayed(const Duration(milliseconds: 40)); + + expect(recovery.recentFailureCount, 0); + expect(watchdog.isRunning, isTrue); + + watchdog.stop(); + await handle.dispose(); + }); + + test('ping timeout marks deadlock and SIGKILLs process', () async { + final process = _FakeProcess(); + final handle = await _handle(process, tempBase); + final recovery = SandboxAutoRecovery(); + final stopped = Completer(); + + final watchdog = SandboxWatchdog( + pingInterval: const Duration(milliseconds: 20), + pongTimeout: const Duration(milliseconds: 30), + recovery: recovery, + onStopped: stopped.complete, + ping: () => Future.delayed(const Duration(seconds: 5), () => 'pong'), + ); + + watchdog.start(handle); + final reason = await stopped.future.timeout(const Duration(seconds: 2)); + + expect(reason, SandboxWatchdogStopReason.deadlock); + expect(process.killed, isTrue); + expect(process.lastSignal, ProcessSignal.sigkill); + expect(recovery.recentFailureCount, 1); + expect(watchdog.isRunning, isFalse); + + await handle.dispose(); + }); + + test('unexpected process exit records failure and stops', () async { + final process = _FakeProcess(); + final handle = await _handle(process, tempBase); + final recovery = SandboxAutoRecovery(); + final stopped = Completer(); + + final watchdog = SandboxWatchdog( + pingInterval: const Duration(hours: 1), + recovery: recovery, + onStopped: stopped.complete, + ping: () async => 'pong', + ); + + watchdog.start(handle); + process.completeExit(1); + final reason = await stopped.future.timeout(const Duration(seconds: 2)); + + expect(reason, SandboxWatchdogStopReason.processExited); + expect(recovery.recentFailureCount, 1); + + await handle.dispose(); + }); + + test('stop cancels timer without killing process', () async { + final process = _FakeProcess(); + final handle = await _handle(process, tempBase); + var pings = 0; + + final watchdog = SandboxWatchdog( + pingInterval: const Duration(milliseconds: 15), + pongTimeout: const Duration(seconds: 1), + ping: () async { + pings++; + return 'pong'; + }, + ); + + watchdog.start(handle); + await Future.delayed(const Duration(milliseconds: 40)); + final before = pings; + watchdog.stop(); + await Future.delayed(const Duration(milliseconds: 50)); + + expect(pings, before); + expect(process.killed, isFalse); + expect(watchdog.lastStopReason, SandboxWatchdogStopReason.stopped); + + await handle.dispose(); + }); + }); + + group('SandboxWatchdog.isPong', () { + test('accepts common result shapes', () { + expect(SandboxWatchdog.isPong(null), isTrue); + expect(SandboxWatchdog.isPong('pong'), isTrue); + expect(SandboxWatchdog.isPong(true), isTrue); + expect(SandboxWatchdog.isPong({'pong': true}), isTrue); + expect(SandboxWatchdog.isPong({'status': 'ok'}), isTrue); + }); + }); +}