diff --git a/benchmarker/README.md b/benchmarker/README.md index 1ce9fcc..753c734 100644 --- a/benchmarker/README.md +++ b/benchmarker/README.md @@ -85,13 +85,29 @@ roughly four times a second: - Outcome breakdown: verified, unverified (no `Content-Length`), byte mismatches, HTTP errors, and failures. +- When the run started and ended, the wall-clock time between them, and the + **build mode** (debug, profile or release) it was measured in. Debug numbers + are not comparable with profile or release ones. + Every response's received byte count is checked against its `Content-Length`; a mismatch is reported as a problem rather than a success. -The copy button at the top right of the statistics panel puts the whole run on -the clipboard — source URL, target URL, cache type, request range, client and -concurrency alongside the numbers — as either an aligned plain-text summary or -JSON for feeding into other tooling. +### Result history + +Every run that reaches a terminal phase — finished, cancelled or failed — is +recorded and stays available from the **Result** dropdown at the top of the +statistics panel, newest first, so runs can be compared without re-running them. +The panel follows the run in flight unless an older one is picked. History lives +in memory only: it is gone when the app exits. + +The copy button at the top right of the statistics panel puts the run on show on +the clipboard — source URL, target URL, cache type, request range, client, +concurrency, timestamps and build mode alongside the numbers — as either an +aligned plain-text summary or JSON. Its third option exports **every** recorded +run as a JSON list for feeding into other tooling. + +The clear button beside it deletes the run on show, or clears the whole history. +A run still in flight has not been recorded yet, so it cannot be deleted. For cache-server runs, the page also shows live download progress (`x / y bytes`, percentage) taken from `HttpCacheStream.cacheStateStream`, and @@ -107,6 +123,7 @@ lib/ benchmark_config.dart inputs, run types, request distribution benchmark_controller.dart run orchestration and aggregation benchmark_report.dart clipboard reports (text and JSON) + benchmark_result.dart one recorded run: timings, mode, stats benchmark_stats.dart timing/percentile accumulation benchmark_worker.dart worker isolate entry point http_client_builder.dart selectable http client implementations diff --git a/benchmarker/lib/src/benchmark/benchmark_controller.dart b/benchmarker/lib/src/benchmark/benchmark_controller.dart index abe3dbd..98ccc62 100644 --- a/benchmarker/lib/src/benchmark/benchmark_controller.dart +++ b/benchmarker/lib/src/benchmark/benchmark_controller.dart @@ -8,23 +8,35 @@ import '../util/formatting.dart'; import 'benchmark_config.dart'; import 'benchmark_log.dart'; import 'benchmark_report.dart'; +import 'benchmark_result.dart'; import 'benchmark_stats.dart'; import 'worker_pool.dart'; import 'worker_protocol.dart'; enum BenchmarkPhase { - idle, - preparing, - running, - cancelling, - finished, - cancelled, - failed; + idle('Idle'), + preparing('Preparing…'), + running('Running'), + cancelling('Cancelling…'), + finished('Finished'), + cancelled('Cancelled'), + failed('Failed'); + + const BenchmarkPhase(this.label); + + /// Human-readable name of the phase. + final String label; bool get isBusy => this == BenchmarkPhase.preparing || this == BenchmarkPhase.running || this == BenchmarkPhase.cancelling; + + /// Whether a run in this phase is over and can be recorded. + bool get isTerminal => + this == BenchmarkPhase.finished || + this == BenchmarkPhase.cancelled || + this == BenchmarkPhase.failed; } /// Drives a benchmark run: prepares the cache, dispatches work to the isolate @@ -40,6 +52,9 @@ class BenchmarkController extends ChangeNotifier { final Map _problemCounts = {}; final Stopwatch _runClock = Stopwatch(); + /// Completed runs of this session, oldest first. + final List _results = []; + WorkerPool? _pool; StreamSubscription? _poolEvents; HttpCacheStream? _cacheStream; @@ -50,6 +65,7 @@ class BenchmarkController extends ChangeNotifier { final Set _outstandingWorkers = {}; int _jobId = 0; + int _runId = 0; bool _cancelRequested = false; bool _dirty = false; bool _warnedUnverified = false; @@ -61,6 +77,11 @@ class BenchmarkController extends ChangeNotifier { BenchmarkStats? _stats; CacheState? _cacheState; Uri? _targetUrl; + DateTime? _startedAt; + + /// Id of the recorded result the UI is pinned to, or null while it follows + /// the run in flight. + int? _selectedResultId; BenchmarkPhase get phase => _phase; @@ -85,6 +106,76 @@ class BenchmarkController extends ChangeNotifier { /// Number of live worker isolates, or 0 when no pool is spawned. int get poolSize => _pool?.size ?? 0; + /// Status line of the current phase, e.g. `Running · 4 workers`. + String get statusLabel => _phase == BenchmarkPhase.running + ? '${_phase.label} · $poolSize workers' + : _phase.label; + + /// Every completed run of this session, oldest first. + List get results => List.unmodifiable(_results); + + /// The run in flight, or null when no run is in progress. + BenchmarkResult? get liveResult { + final stats = _stats; + final startedAt = _startedAt; + if (!_phase.isBusy || stats == null || startedAt == null) return null; + return BenchmarkResult( + id: _runId, + stats: stats, + status: statusLabel, + startedAt: startedAt, + config: _config, + targetUrl: _targetUrl, + ); + } + + /// The result the UI shows: the pinned one from the history, or the run in + /// flight when nothing is pinned. + BenchmarkResult? get selectedResult { + final id = _selectedResultId; + if (id != null) { + for (final result in _results) { + if (result.id == id) return result; + } + } + return liveResult; + } + + /// Id of the result the UI shows, or null when there is nothing to show. + int? get selectedResultId => selectedResult?.id; + + /// Pins the history entry with [id]. Selecting the run in flight, or an id + /// that is no longer in the history, follows the live run again. + void selectResult(int? id) { + _selectedResultId = + id == null || (liveResult != null && id == liveResult!.id) ? null : id; + notifyListeners(); + } + + /// Removes a single run from the history. + /// + /// A run in flight is never recorded yet, so it cannot be deleted. + void deleteResult(int id) { + final removed = _results.indexWhere((result) => result.id == id); + if (removed < 0) return; + _results.removeAt(removed); + if (_selectedResultId == id) { + // Fall back to the neighbouring run, or to the live run when the history + // is empty. + final next = removed < _results.length ? removed : _results.length - 1; + _selectedResultId = next < 0 ? null : _results[next].id; + } + notifyListeners(); + } + + /// Drops every recorded run. The run in flight, if any, keeps going. + void clearResults() { + if (_results.isEmpty) return; + _results.clear(); + _selectedResultId = null; + notifyListeners(); + } + /// Starts a run. Does nothing when a run is already in progress. Future start(BenchmarkConfig config) async { if (_phase.isBusy || _disposed) return; @@ -98,11 +189,20 @@ class BenchmarkController extends ChangeNotifier { _stats = BenchmarkStats.empty(config.totalRequests); _cacheState = null; _targetUrl = null; + _startedAt = DateTime.now(); + // The stats panel follows the new run rather than whatever the user was + // reading in the history. + _selectedResultId = null; + _runId++; _runClock ..reset() ..stop(); - _log('── ${config.type.label} run: ${config.sourceUrl}'); + _log('── Run #$_runId · ${config.type.label}: ${config.sourceUrl}'); + _log( + 'Started ${formatTimestamp(_startedAt!)} · ' + '${BuildMode.current.label} build', + ); _log( '${config.totalRequests} requests · ${config.concurrency} workers · ' '${config.clientOption.label}', @@ -296,10 +396,37 @@ class BenchmarkController extends ChangeNotifier { _stopTicker(); _refreshStats(); _summarize(phase); + _record(phase); unawaited(_releaseCacheStream()); _setPhase(phase); } + /// Files the finished run in the history and pins the stats panel to it, + /// unless the user has pinned an older run in the meantime. + void _record(BenchmarkPhase phase) { + final stats = _stats; + final startedAt = _startedAt; + if (!phase.isTerminal || stats == null || startedAt == null) return; + + final endedAt = DateTime.now(); + final result = BenchmarkResult( + id: _runId, + stats: stats, + status: phase.label, + startedAt: startedAt, + endedAt: endedAt, + config: _config, + targetUrl: _targetUrl, + ); + _results.add(result); + _selectedResultId ??= result.id; + + _log( + 'Ended ${formatTimestamp(endedAt)} · ' + '${formatDuration(result.wallDuration!)} wall clock', + ); + } + void _summarize(BenchmarkPhase phase) { final stats = _stats; if (stats == null) return; diff --git a/benchmarker/lib/src/benchmark/benchmark_report.dart b/benchmarker/lib/src/benchmark/benchmark_report.dart index 70510c1..e85887f 100644 --- a/benchmarker/lib/src/benchmark/benchmark_report.dart +++ b/benchmarker/lib/src/benchmark/benchmark_report.dart @@ -2,8 +2,11 @@ import 'dart:convert'; import '../util/formatting.dart'; import 'benchmark_config.dart'; +import 'benchmark_result.dart'; import 'benchmark_stats.dart'; +const JsonEncoder _encoder = JsonEncoder.withIndent(' '); + /// One line describing what each request in [config] asks for. String describeRangePlan(BenchmarkConfig config) { final plan = config.rangePlan; @@ -17,12 +20,18 @@ String describeRangePlan(BenchmarkConfig config) { } /// Renders a run's configuration and results as indented JSON. -String buildJsonReport({ - required BenchmarkStats stats, - BenchmarkConfig? config, - Uri? targetUrl, - String? status, -}) { +String buildJsonReport(BenchmarkResult result) => + _encoder.convert(buildReportMap(result)); + +/// Renders every run as a JSON list, oldest first. +String buildJsonReportList(Iterable results) => + _encoder.convert([for (final result in results) buildReportMap(result)]); + +/// Builds the JSON structure describing a single run. +Map buildReportMap(BenchmarkResult result) { + final stats = result.stats; + final config = result.config; + Map timing(TimingStats? timing) { if (timing == null) return {}; return { @@ -37,12 +46,17 @@ String buildJsonReport({ } final plan = config?.rangePlan; - final report = { + return { + 'run_id': result.id, 'source_url': config?.sourceUrl.toString(), - 'target_url': targetUrl?.toString(), + 'target_url': result.targetUrl?.toString(), 'cache_type': config?.type.name, 'cache_type_label': config?.type.label, - 'status': status, + 'status': result.status, + 'build_mode': result.mode.name, + 'started_at': result.startedAt.toIso8601String(), + 'ended_at': result.endedAt?.toIso8601String(), + 'wall_duration_us': result.wallDuration?.inMicroseconds, 'concurrency': config?.concurrency, 'http_client': config?.clientOption.label, 'range': { @@ -82,21 +96,16 @@ String buildJsonReport({ 'completion': timing(stats.completionTime), }, }; - - return const JsonEncoder.withIndent(' ').convert(report); } /// Renders a run's configuration and results as an aligned plain-text summary. -String buildTextReport({ - required BenchmarkStats stats, - BenchmarkConfig? config, - Uri? targetUrl, - String? status, -}) { +String buildTextReport(BenchmarkResult result) { + final stats = result.stats; + final config = result.config; final buffer = StringBuffer() ..writeln( 'http_cache_stream benchmark' - '${config == null ? '' : ' — ${config.type.label}'}', + '${config == null ? '' : ' — ${config.type.label}'} (run #${result.id})', ); void field(String label, String? value) { @@ -105,14 +114,24 @@ String buildTextReport({ } field('Source', config?.sourceUrl.toString()); - field('Target', targetUrl?.toString()); + field('Target', result.targetUrl?.toString()); field('Client', config?.clientOption.label); field( 'Concurrency', config == null ? null : '${config.concurrency} worker isolates', ); field('Range', config == null ? null : describeRangePlan(config)); - field('Status', status); + field('Status', result.status); + field('Mode', '${result.mode.label} build'); + field('Started', formatTimestamp(result.startedAt)); + field( + 'Ended', + result.endedAt == null ? null : formatTimestamp(result.endedAt!), + ); + field( + 'Duration', + result.wallDuration == null ? null : formatDuration(result.wallDuration!), + ); buffer ..writeln() diff --git a/benchmarker/lib/src/benchmark/benchmark_result.dart b/benchmarker/lib/src/benchmark/benchmark_result.dart new file mode 100644 index 0000000..bf34990 --- /dev/null +++ b/benchmarker/lib/src/benchmark/benchmark_result.dart @@ -0,0 +1,88 @@ +import 'package:flutter/foundation.dart'; + +import '../util/formatting.dart'; +import 'benchmark_config.dart'; +import 'benchmark_stats.dart'; + +/// The Flutter build mode the benchmarker itself was compiled in. +/// +/// Results are only comparable between runs of the same mode: debug builds pay +/// for assertions and an unoptimised VM, so their numbers are not +/// representative of a shipped app. +enum BuildMode { + debug('Debug'), + profile('Profile'), + release('Release'); + + const BuildMode(this.label); + + final String label; + + /// The mode this binary is running in. + static BuildMode get current { + if (kDebugMode) return BuildMode.debug; + if (kProfileMode) return BuildMode.profile; + return BuildMode.release; + } +} + +/// One benchmark run: its inputs, its results, and when it ran. +/// +/// The controller builds a live instance while a run is in flight and keeps a +/// final one in its history once the run reaches a terminal phase. +class BenchmarkResult { + BenchmarkResult({ + required this.id, + required this.stats, + required this.status, + required this.startedAt, + this.endedAt, + this.config, + this.targetUrl, + BuildMode? mode, + }) : mode = mode ?? BuildMode.current; + + /// Run number, counting from 1 within the session. + final int id; + + /// Results as of the last snapshot taken. + final BenchmarkStats stats; + + /// Human-readable phase of the run, e.g. `Finished` or `Running · 4 workers`. + final String status; + + /// Wall-clock time the run was started, before any preparation. + final DateTime startedAt; + + /// Wall-clock time the run reached a terminal phase, or null while running. + final DateTime? endedAt; + + /// Inputs of the run. + final BenchmarkConfig? config; + + /// URL the workers hit: the cache URL, or the source URL for direct runs. + final Uri? targetUrl; + + /// Build mode the run was measured in. + final BuildMode mode; + + /// Whether the run has reached a terminal phase. + bool get isComplete => endedAt != null; + + /// Wall-clock time from start to end, including cache preparation. Null while + /// the run is still in flight. + /// + /// This is longer than [BenchmarkStats.elapsed], which only covers the + /// measured requests. + Duration? get wallDuration => endedAt?.difference(startedAt); + + /// Label of the benchmark type, or a placeholder when the run has no config. + String get typeLabel => config?.type.label ?? 'Run'; + + /// Single line identifying the run in the history dropdown. + String get label => '#$id · $typeLabel · ${formatClockTime(startedAt)}'; + + /// Second line of the history dropdown: how the run went. + String get detail => '$status · ${mode.label} · ${stats.completed}/' + '${stats.totalRequests} requests · ${formatDuration(stats.elapsed)}'; +} diff --git a/benchmarker/lib/src/ui/benchmark_page.dart b/benchmarker/lib/src/ui/benchmark_page.dart index 4b0824f..8b1d06d 100644 --- a/benchmarker/lib/src/ui/benchmark_page.dart +++ b/benchmarker/lib/src/ui/benchmark_page.dart @@ -29,26 +29,6 @@ class _BenchmarkPageState extends State { super.dispose(); } - String get _status { - final controller = _controller; - switch (controller.phase) { - case BenchmarkPhase.idle: - return 'Idle'; - case BenchmarkPhase.preparing: - return 'Preparing…'; - case BenchmarkPhase.running: - return 'Running · ${controller.poolSize} workers'; - case BenchmarkPhase.cancelling: - return 'Cancelling…'; - case BenchmarkPhase.finished: - return 'Finished'; - case BenchmarkPhase.cancelled: - return 'Cancelled'; - case BenchmarkPhase.failed: - return 'Failed'; - } - } - @override Widget build(BuildContext context) { return Scaffold( @@ -91,10 +71,12 @@ class _BenchmarkPageState extends State { ) : null; final stats = StatsPanel( - stats: _controller.stats, - status: _status, - config: _controller.config, - targetUrl: _controller.targetUrl, + result: _controller.selectedResult, + status: _controller.statusLabel, + history: _controller.results, + onSelect: _controller.selectResult, + onDelete: _controller.deleteResult, + onClearAll: _controller.clearResults, ); final log = LogPanel( logs: _controller.logs, diff --git a/benchmarker/lib/src/ui/widgets/stats_panel.dart b/benchmarker/lib/src/ui/widgets/stats_panel.dart index b36ff23..139b0e3 100644 --- a/benchmarker/lib/src/ui/widgets/stats_panel.dart +++ b/benchmarker/lib/src/ui/widgets/stats_panel.dart @@ -1,90 +1,153 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; -import '../../benchmark/benchmark_config.dart'; import '../../benchmark/benchmark_report.dart'; +import '../../benchmark/benchmark_result.dart'; import '../../benchmark/benchmark_stats.dart'; import '../../util/formatting.dart'; import 'section_card.dart'; /// Clipboard formats offered by the copy button. -enum _CopyFormat { json, text } +enum _CopyFormat { json, text, allJson } -/// Aggregated results of the current or most recent run. +/// Options offered by the clear button. +enum _ClearAction { current, all } + +/// Aggregated results of the selected run, with the session's past runs behind +/// a dropdown. class StatsPanel extends StatelessWidget { const StatsPanel({ super.key, - required this.stats, + required this.result, required this.status, - this.config, - this.targetUrl, + this.history = const [], + this.onSelect, + this.onDelete, + this.onClearAll, }); - final BenchmarkStats? stats; + /// The run on show: the one in flight, or a past one picked from [history]. + final BenchmarkResult? result; - /// Short status line shown next to the title, e.g. `Running`. + /// Short status line shown when there is no result yet, e.g. `Idle`. final String status; - /// Inputs of the run the stats belong to, copied alongside them. - final BenchmarkConfig? config; + /// Every completed run of this session, oldest first. + final List history; + + /// Called with the id of the run to show. + final ValueChanged? onSelect; + + /// Called with the id of the run to drop from [history]. + final ValueChanged? onDelete; - /// URL the workers hit: the cache URL, or the source URL for direct runs. - final Uri? targetUrl; + /// Called to drop every run from [history]. + final VoidCallback? onClearAll; + + /// Whether the run on show has already been recorded, and so can be deleted. + bool get _isRecorded { + final result = this.result; + return result != null && history.any((entry) => entry.id == result.id); + } void _copy(BuildContext context, _CopyFormat format) { - final stats = this.stats; - if (stats == null) return; - final report = switch (format) { - _CopyFormat.json => buildJsonReport( - stats: stats, - config: config, - targetUrl: targetUrl, - status: status, - ), - _CopyFormat.text => buildTextReport( - stats: stats, - config: config, - targetUrl: targetUrl, - status: status, - ), - }; + final result = this.result; + final String report; + final String message; + switch (format) { + case _CopyFormat.text: + if (result == null) return; + report = buildTextReport(result); + message = 'Statistics copied as text.'; + case _CopyFormat.json: + if (result == null) return; + report = buildJsonReport(result); + message = 'Statistics copied as JSON.'; + case _CopyFormat.allJson: + if (history.isEmpty) return; + report = buildJsonReportList(history); + message = '${history.length} result(s) copied as a JSON list.'; + } Clipboard.setData(ClipboardData(text: report)); ScaffoldMessenger.of(context).showSnackBar( - SnackBar( - content: Text( - format == _CopyFormat.json - ? 'Statistics copied as JSON.' - : 'Statistics copied as text.', - ), - ), + SnackBar(content: Text(message)), + ); + } + + void _clear(BuildContext context, _ClearAction action) { + final String message; + switch (action) { + case _ClearAction.current: + final result = this.result; + if (result == null || !_isRecorded) return; + onDelete?.call(result.id); + message = 'Result #${result.id} deleted.'; + case _ClearAction.all: + if (history.isEmpty) return; + message = '${history.length} result(s) cleared.'; + onClearAll?.call(); + } + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text(message)), ); } @override Widget build(BuildContext context) { final theme = Theme.of(context); - final stats = this.stats; + final result = this.result; + final stats = result?.stats; return SectionCard( title: 'Statistics', - subtitle: status, - trailing: PopupMenuButton<_CopyFormat>( - enabled: stats != null, - tooltip: 'Copy statistics', - icon: const Icon(Icons.copy_all_outlined), - onSelected: (format) => _copy(context, format), - itemBuilder: (context) => const [ - PopupMenuItem<_CopyFormat>( - value: _CopyFormat.text, - child: Text('Copy as text'), + subtitle: result == null ? status : result.status, + trailing: Row( + mainAxisSize: MainAxisSize.min, + children: [ + PopupMenuButton<_CopyFormat>( + enabled: result != null || history.isNotEmpty, + tooltip: 'Copy statistics', + icon: const Icon(Icons.copy_all_outlined), + onSelected: (format) => _copy(context, format), + itemBuilder: (context) => [ + PopupMenuItem<_CopyFormat>( + value: _CopyFormat.text, + enabled: result != null, + child: const Text('Copy as text'), + ), + PopupMenuItem<_CopyFormat>( + value: _CopyFormat.json, + enabled: result != null, + child: const Text('Copy as JSON'), + ), + PopupMenuItem<_CopyFormat>( + value: _CopyFormat.allJson, + enabled: history.isNotEmpty, + child: Text('Export all results as JSON (${history.length})'), + ), + ], ), - PopupMenuItem<_CopyFormat>( - value: _CopyFormat.json, - child: Text('Copy as JSON'), + PopupMenuButton<_ClearAction>( + enabled: history.isNotEmpty, + tooltip: 'Clear results', + icon: const Icon(Icons.delete_outline), + onSelected: (action) => _clear(context, action), + itemBuilder: (context) => [ + PopupMenuItem<_ClearAction>( + value: _ClearAction.current, + enabled: _isRecorded, + child: const Text('Delete current result'), + ), + PopupMenuItem<_ClearAction>( + value: _ClearAction.all, + enabled: history.isNotEmpty, + child: const Text('Clear all results'), + ), + ], ), ], ), - child: stats == null + child: result == null || stats == null ? Padding( padding: const EdgeInsets.symmetric(vertical: 24), child: Text( @@ -97,6 +160,14 @@ class StatsPanel extends StatelessWidget { : Column( crossAxisAlignment: CrossAxisAlignment.stretch, children: [ + _ResultSelector( + result: result, + history: history, + onSelect: onSelect, + ), + const SizedBox(height: 12), + _RunMeta(result: result), + const SizedBox(height: 12), ClipRRect( borderRadius: BorderRadius.circular(4), child: LinearProgressIndicator( @@ -116,6 +187,122 @@ class StatsPanel extends StatelessWidget { } } +/// Dropdown listing the session's runs, most recent first, so a past result can +/// be brought back into the panel. +class _ResultSelector extends StatelessWidget { + const _ResultSelector({ + required this.result, + required this.history, + this.onSelect, + }); + + final BenchmarkResult result; + final List history; + final ValueChanged? onSelect; + + @override + Widget build(BuildContext context) { + // A run in flight is not recorded yet, so it is listed on top of the + // history rather than taken from it. + final entries = [ + if (!history.any((entry) => entry.id == result.id)) result, + ...history.reversed, + ]; + + return InputDecorator( + decoration: InputDecoration( + labelText: 'Result', + helperText: entries.length == 1 + ? 'Completed runs are kept here for this session.' + : '${entries.length} runs this session', + border: const OutlineInputBorder(), + isDense: true, + ), + child: DropdownButtonHideUnderline( + child: DropdownButton( + value: result.id, + isExpanded: true, + isDense: true, + onChanged: onSelect == null || entries.length < 2 + ? null + : (id) { + if (id != null) onSelect!(id); + }, + selectedItemBuilder: (context) => [ + for (final entry in entries) + Align( + alignment: Alignment.centerLeft, + child: Text(entry.label, overflow: TextOverflow.ellipsis), + ), + ], + items: [ + for (final entry in entries) + DropdownMenuItem( + value: entry.id, + child: _ResultEntry(result: entry), + ), + ], + ), + ), + ); + } +} + +/// Two-line description of a run inside the dropdown. +class _ResultEntry extends StatelessWidget { + const _ResultEntry({required this.result}); + + final BenchmarkResult result; + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Text(result.label, overflow: TextOverflow.ellipsis), + Text( + result.detail, + overflow: TextOverflow.ellipsis, + style: theme.textTheme.bodySmall?.copyWith( + color: theme.colorScheme.onSurfaceVariant, + ), + ), + ], + ); + } +} + +/// When the run ran, and in which build mode. +class _RunMeta extends StatelessWidget { + const _RunMeta({required this.result}); + + final BenchmarkResult result; + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final ended = result.endedAt; + return DefaultTextStyle.merge( + style: theme.textTheme.bodySmall?.copyWith( + color: theme.colorScheme.onSurfaceVariant, + ), + child: Wrap( + spacing: 12, + runSpacing: 4, + children: [ + Text('${result.mode.label} build'), + Text('Started ${formatTimestamp(result.startedAt)}'), + if (ended != null) Text('Ended ${formatTimestamp(ended)}'), + if (result.wallDuration case final duration?) + Text('Wall clock ${formatDuration(duration)}'), + ], + ), + ); + } +} + class _TileGrid extends StatelessWidget { const _TileGrid({required this.stats}); diff --git a/benchmarker/lib/src/util/formatting.dart b/benchmarker/lib/src/util/formatting.dart index 61ee429..7fae8d9 100644 --- a/benchmarker/lib/src/util/formatting.dart +++ b/benchmarker/lib/src/util/formatting.dart @@ -45,8 +45,17 @@ String formatPercent(double fraction, {int fractionDigits = 1}) => /// Formats a wall-clock time as `HH:mm:ss.SSS`. String formatClockTime(DateTime time) { - String pad(int value, [int width = 2]) => - value.toString().padLeft(width, '0'); - return '${pad(time.hour)}:${pad(time.minute)}:${pad(time.second)}' - '.${pad(time.millisecond, 3)}'; + return '${_pad(time.hour)}:${_pad(time.minute)}:${_pad(time.second)}' + '.${_pad(time.millisecond, 3)}'; } + +/// Formats a date and wall-clock time as `yyyy-MM-dd HH:mm:ss.SSS`, in the +/// local time zone. +String formatTimestamp(DateTime time) { + final local = time.toLocal(); + return '${local.year}-${_pad(local.month)}-${_pad(local.day)} ' + '${formatClockTime(local)}'; +} + +String _pad(int value, [int width = 2]) => + value.toString().padLeft(width, '0'); diff --git a/benchmarker/test/benchmark_controller_test.dart b/benchmarker/test/benchmark_controller_test.dart index 567b446..ecccfa2 100644 --- a/benchmarker/test/benchmark_controller_test.dart +++ b/benchmarker/test/benchmark_controller_test.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:benchmarker/src/benchmark/benchmark_config.dart'; import 'package:benchmarker/src/benchmark/benchmark_controller.dart'; +import 'package:benchmarker/src/benchmark/benchmark_result.dart'; import 'package:benchmarker/src/benchmark/http_client_builder.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:http_cache_stream/http_cache_stream.dart'; @@ -238,6 +239,67 @@ void main() { expect(originRequests, 1); }); + test('completed runs are kept, selectable and removable', () async { + expect(controller.results, isEmpty); + expect(controller.selectedResult, isNull); + + final before = DateTime.now(); + await controller.start(configFor(BenchmarkType.direct, total: 2)); + await controller.start(configFor(BenchmarkType.direct, total: 2)); + final after = DateTime.now(); + + expect(controller.results.map((result) => result.id), [1, 2]); + + final first = controller.results.first; + expect(first.status, 'Finished'); + expect(first.stats.completed, 2); + expect(first.config!.type, BenchmarkType.direct); + expect(first.targetUrl, sourceUrl); + expect(first.mode, BuildMode.current); + expect(first.isComplete, isTrue); + expect(first.startedAt.isBefore(first.endedAt!), isTrue); + expect(first.startedAt.isBefore(before), isFalse); + expect(first.endedAt!.isAfter(after), isFalse); + // The wall clock covers preparation as well as the measured requests. + expect( + first.wallDuration!.inMicroseconds, + greaterThanOrEqualTo(first.stats.elapsed.inMicroseconds), + ); + + // The panel follows the newest run until an older one is picked. + expect(controller.selectedResultId, 2); + controller.selectResult(1); + expect(controller.selectedResultId, 1); + + controller.deleteResult(1); + expect(controller.results.map((result) => result.id), [2]); + expect(controller.selectedResultId, 2); + + controller.clearResults(); + expect(controller.results, isEmpty); + expect(controller.selectedResult, isNull); + }); + + test('a run in flight is selectable before it is recorded', () async { + final run = controller.start(configFor(BenchmarkType.direct, total: 4)); + + // start() sets the phase before its first suspension, so the run is + // already in flight here. + expect(controller.phase.isBusy, isTrue); + final live = controller.liveResult!; + expect(live.id, 1); + expect(live.isComplete, isFalse); + expect(live.wallDuration, isNull); + expect(live.status, controller.statusLabel); + expect(controller.results, isEmpty); + + await run; + + expect(controller.liveResult, isNull); + expect(controller.results.single.id, live.id); + expect(controller.selectedResult!.isComplete, isTrue); + }); + test('the worker pool is reused between runs with the same settings', () async { await controller.start(configFor(BenchmarkType.direct, total: 2)); diff --git a/benchmarker/test/benchmark_report_test.dart b/benchmarker/test/benchmark_report_test.dart index 8fb44ad..82a282c 100644 --- a/benchmarker/test/benchmark_report_test.dart +++ b/benchmarker/test/benchmark_report_test.dart @@ -2,11 +2,36 @@ import 'dart:convert'; import 'package:benchmarker/src/benchmark/benchmark_config.dart'; import 'package:benchmarker/src/benchmark/benchmark_report.dart'; +import 'package:benchmarker/src/benchmark/benchmark_result.dart'; import 'package:benchmarker/src/benchmark/benchmark_stats.dart'; import 'package:benchmarker/src/benchmark/http_client_builder.dart'; import 'package:benchmarker/src/benchmark/worker_protocol.dart'; +import 'package:benchmarker/src/util/formatting.dart'; import 'package:flutter_test/flutter_test.dart'; +final DateTime _startedAt = DateTime.utc(2024, 5, 6, 7, 8, 9, 10); +final DateTime _endedAt = _startedAt.add(const Duration(seconds: 5)); + +BenchmarkResult _result({ + int id = 1, + BenchmarkStats? stats, + BenchmarkConfig? config, + Uri? targetUrl, + String status = 'Finished', + bool inFlight = false, +}) { + return BenchmarkResult( + id: id, + stats: stats ?? _stats(), + status: status, + startedAt: _startedAt, + endedAt: inFlight ? null : _endedAt, + config: config, + targetUrl: targetUrl, + mode: BuildMode.release, + ); +} + BenchmarkConfig _config({RangePlan? rangePlan, int total = 4}) { return BenchmarkConfig( sourceUrl: Uri.parse('https://example.com/file.bin'), @@ -70,20 +95,27 @@ void main() { test('carries the run inputs and results', () { final json = jsonDecode( buildJsonReport( - stats: _stats(), - config: _config( - rangePlan: RangePlan.sequential(const ByteRange(0, 4095), 4), + _result( + id: 3, + config: _config( + rangePlan: RangePlan.sequential(const ByteRange(0, 4095), 4), + ), + targetUrl: + Uri.parse('http://127.0.0.1:4612/https/example.com/f.bin'), ), - targetUrl: Uri.parse('http://127.0.0.1:4612/https/example.com/f.bin'), - status: 'Finished', ), ) as Map; + expect(json['run_id'], 3); expect(json['source_url'], 'https://example.com/file.bin'); expect(json['target_url'], 'http://127.0.0.1:4612/https/example.com/f.bin'); expect(json['cache_type'], 'preCached'); expect(json['cache_type_label'], 'Pre-cached'); expect(json['status'], 'Finished'); + expect(json['build_mode'], 'release'); + expect(json['started_at'], _startedAt.toIso8601String()); + expect(json['ended_at'], _endedAt.toIso8601String()); + expect(json['wall_duration_us'], const Duration(seconds: 5).inMicroseconds); expect(json['concurrency'], 2); expect(json['http_client'], kHttpClientOptions.first.label); @@ -114,7 +146,7 @@ void main() { test('marks a full-response run and omits range bounds', () { final json = jsonDecode( - buildJsonReport(stats: _stats(), config: _config()), + buildJsonReport(_result(config: _config())), ) as Map; final range = json['range']! as Map; @@ -123,43 +155,100 @@ void main() { }); test('renders without a config', () { - final json = jsonDecode(buildJsonReport(stats: _stats())) + final json = jsonDecode(buildJsonReport(_result())) as Map; expect(json['source_url'], isNull); expect((json['requests']! as Map)['completed'], 4); }); + + test('leaves the end open for a run still in flight', () { + final json = jsonDecode( + buildJsonReport( + _result(status: 'Running · 2 workers', inFlight: true), + ), + ) as Map; + + expect(json['status'], 'Running · 2 workers'); + expect(json['ended_at'], isNull); + expect(json['wall_duration_us'], isNull); + }); + }); + + group('buildJsonReportList', () { + test('renders every run as a JSON list, oldest first', () { + final list = jsonDecode( + buildJsonReportList([ + _result(id: 1, config: _config()), + _result(id: 2, config: _config(), stats: _stats(completed: 2)), + ]), + ) as List; + + expect(list, hasLength(2)); + expect((list[0]! as Map)['run_id'], 1); + expect((list[1]! as Map)['run_id'], 2); + expect( + ((list[1]! as Map)['requests']! + as Map)['completed'], + 2, + ); + }); + + test('renders an empty history as an empty list', () { + expect(jsonDecode(buildJsonReportList(const [])), isEmpty); + }); }); group('buildTextReport', () { test('lists the run inputs above the results', () { final text = buildTextReport( - stats: _stats(), - config: _config( - rangePlan: RangePlan.fixed(const ByteRange(1024, 5119)), + _result( + id: 7, + config: _config( + rangePlan: RangePlan.fixed(const ByteRange(1024, 5119)), + ), + targetUrl: Uri.parse('http://127.0.0.1:4612/https/example.com/f.bin'), ), - targetUrl: Uri.parse('http://127.0.0.1:4612/https/example.com/f.bin'), - status: 'Finished', ); - expect(text, contains('http_cache_stream benchmark — Pre-cached')); + expect( + text, + contains('http_cache_stream benchmark — Pre-cached (run #7)'), + ); expect(text, contains('Source: https://example.com/file.bin')); expect(text, contains('Target: http://127.0.0.1:4612/')); expect(text, contains('Client: ${kHttpClientOptions.first.label}')); expect(text, contains('Concurrency: 2 worker isolates')); expect(text, contains('Range: Fixed range bytes=1024-5119')); expect(text, contains('Status: Finished')); + expect(text, contains('Mode: Release build')); + expect(text, contains('Started: ${formatTimestamp(_startedAt)}')); + expect(text, contains('Ended: ${formatTimestamp(_endedAt)}')); + expect(text, contains('Duration: 5.00 s')); expect(text, contains('Requests: 4 / 4 completed · 4 verified')); expect(text, contains('Elapsed: 2.00 s')); expect(text, contains('Throughput: 2.00 req/s')); expect(text, contains('Bytes: 4.00 KB total')); }); + test('omits the end of a run still in flight', () { + final text = buildTextReport( + _result(status: 'Running · 2 workers', inFlight: true), + ); + + expect(text, contains('Status: Running · 2 workers')); + expect(text, contains('Started: ')); + expect(text, isNot(contains('Ended:'))); + expect(text, isNot(contains('Duration:'))); + }); + test('aligns the timing table and marks missing series', () { final text = buildTextReport( - stats: const BenchmarkStats.empty(10), - config: _config(), - status: 'Idle', + _result( + stats: const BenchmarkStats.empty(10), + config: _config(), + status: 'Idle', + ), ); final lines = text.split('\n'); diff --git a/benchmarker/test/stats_panel_test.dart b/benchmarker/test/stats_panel_test.dart index a9ec2d2..9d8eede 100644 --- a/benchmarker/test/stats_panel_test.dart +++ b/benchmarker/test/stats_panel_test.dart @@ -1,6 +1,7 @@ import 'dart:convert'; import 'package:benchmarker/src/benchmark/benchmark_config.dart'; +import 'package:benchmarker/src/benchmark/benchmark_result.dart'; import 'package:benchmarker/src/benchmark/benchmark_stats.dart'; import 'package:benchmarker/src/benchmark/http_client_builder.dart'; import 'package:benchmarker/src/benchmark/worker_protocol.dart'; @@ -18,6 +19,8 @@ void main() { clientOption: kHttpClientOptions.first, rangePlan: RangePlan.sequential(const ByteRange(0, 2047), 2), ); + final targetUrl = Uri.parse('http://127.0.0.1:4612/https/example.com/f'); + final startedAt = DateTime(2024, 5, 6, 7, 8, 9); BenchmarkStats statsFor(int completed) { final accumulator = StatsAccumulator(2); @@ -39,6 +42,25 @@ void main() { return accumulator.snapshot(const Duration(seconds: 1)); } + BenchmarkResult resultFor( + int id, { + int completed = 2, + String status = 'Finished', + bool inFlight = false, + }) { + final start = startedAt.add(Duration(minutes: id)); + return BenchmarkResult( + id: id, + stats: statsFor(completed), + status: status, + startedAt: start, + endedAt: inFlight ? null : start.add(const Duration(seconds: 3)), + config: config, + targetUrl: targetUrl, + mode: BuildMode.debug, + ); + } + /// Captures whatever the panel writes to the clipboard. String? copied; @@ -58,16 +80,25 @@ void main() { .setMockMethodCallHandler(SystemChannels.platform, null); }); - Future pumpPanel(WidgetTester tester, {BenchmarkStats? stats}) { + Future pumpPanel( + WidgetTester tester, { + BenchmarkResult? result, + List history = const [], + ValueChanged? onSelect, + ValueChanged? onDelete, + VoidCallback? onClearAll, + }) { return tester.pumpWidget( MaterialApp( home: Scaffold( body: SingleChildScrollView( child: StatsPanel( - stats: stats, - status: 'Finished', - config: config, - targetUrl: Uri.parse('http://127.0.0.1:4612/https/example.com/f'), + result: result, + status: 'Idle', + history: history, + onSelect: onSelect, + onDelete: onDelete, + onClearAll: onClearAll, ), ), ), @@ -75,11 +106,16 @@ void main() { ); } + Future openMenu(WidgetTester tester, IconData icon) async { + await tester.tap(find.byIcon(icon)); + await tester.pumpAndSettle(); + } + testWidgets('copies the run and its statistics as text', (tester) async { - await pumpPanel(tester, stats: statsFor(2)); + final result = resultFor(1); + await pumpPanel(tester, result: result, history: [result]); - await tester.tap(find.byIcon(Icons.copy_all_outlined)); - await tester.pumpAndSettle(); + await openMenu(tester, Icons.copy_all_outlined); await tester.tap(find.text('Copy as text')); await tester.pumpAndSettle(); @@ -88,38 +124,166 @@ void main() { expect(copied, contains('Non-cached')); expect(copied, contains('Sequential windows: 2 × 1.00 KB')); expect(copied, contains('Requests: 2 / 2 completed')); + expect(copied, contains('Mode: Debug build')); + expect(copied, contains('Started: 2024-05-06 07:09:09')); + expect(copied, contains('Ended: 2024-05-06 07:09:12')); expect(copied, contains('Completion')); expect(find.text('Statistics copied as text.'), findsOneWidget); }); testWidgets('copies the run and its statistics as JSON', (tester) async { - await pumpPanel(tester, stats: statsFor(2)); + final result = resultFor(1); + await pumpPanel(tester, result: result, history: [result]); - await tester.tap(find.byIcon(Icons.copy_all_outlined)); - await tester.pumpAndSettle(); + await openMenu(tester, Icons.copy_all_outlined); await tester.tap(find.text('Copy as JSON')); await tester.pumpAndSettle(); final json = jsonDecode(copied!) as Map; + expect(json['run_id'], 1); expect(json['source_url'], 'https://example.com/file.bin'); expect(json['target_url'], 'http://127.0.0.1:4612/https/example.com/f'); expect(json['cache_type'], 'nonCached'); expect(json['status'], 'Finished'); + expect(json['build_mode'], 'debug'); + expect(json['started_at'], isNotNull); + expect(json['ended_at'], isNotNull); expect((json['range']! as Map)['mode'], 'sequential'); expect((json['requests']! as Map)['completed'], 2); expect(find.text('Statistics copied as JSON.'), findsOneWidget); }); - testWidgets('the copy button is disabled before the first run', + testWidgets('exports every recorded result as a JSON list', (tester) async { + final history = [resultFor(1), resultFor(2, completed: 1)]; + await pumpPanel(tester, result: history.last, history: history); + + await openMenu(tester, Icons.copy_all_outlined); + await tester.tap(find.text('Export all results as JSON (2)')); + await tester.pumpAndSettle(); + + final json = jsonDecode(copied!) as List; + expect(json, hasLength(2)); + expect((json.first! as Map)['run_id'], 1); + expect((json.last! as Map)['run_id'], 2); + expect( + ((json.last! as Map)['requests']! as Map)['completed'], + 1, + ); + expect(find.text('2 result(s) copied as a JSON list.'), findsOneWidget); + }); + + testWidgets('lists past runs in the dropdown and reports the pick', (tester) async { - await pumpPanel(tester); + final history = [ + resultFor(1, status: 'Cancelled'), + resultFor(2), + ]; + int? selected; + await pumpPanel( + tester, + result: history.last, + history: history, + onSelect: (id) => selected = id, + ); - final button = tester.widget( - find.ancestor( - of: find.byIcon(Icons.copy_all_outlined), - matching: find.byType(IconButton), - ), + expect(find.text(history.last.label), findsOneWidget); + + await tester.tap(find.byType(DropdownButton)); + await tester.pumpAndSettle(); + // Most recent first, so the older run sits below the current one. + expect(find.text(history.first.detail), findsOneWidget); + + await tester.tap(find.text(history.first.detail)); + await tester.pumpAndSettle(); + + expect(selected, 1); + }); + + testWidgets('lists the run in flight above the recorded ones', + (tester) async { + final history = [resultFor(1)]; + final live = resultFor(2, status: 'Running · 2 workers', inFlight: true); + await pumpPanel( + tester, + result: live, + history: history, + onSelect: (_) {}, + ); + + expect(find.text('Running · 2 workers'), findsOneWidget); + expect(find.text(live.label), findsOneWidget); + expect(find.text('2 runs this session'), findsOneWidget); + }); + + testWidgets('deletes the result on show', (tester) async { + final history = [resultFor(1), resultFor(2)]; + int? deleted; + await pumpPanel( + tester, + result: history.last, + history: history, + onDelete: (id) => deleted = id, + onClearAll: () {}, + ); + + await openMenu(tester, Icons.delete_outline); + await tester.tap(find.text('Delete current result')); + await tester.pumpAndSettle(); + + expect(deleted, 2); + expect(find.text('Result #2 deleted.'), findsOneWidget); + }); + + testWidgets('clears every recorded result', (tester) async { + final history = [resultFor(1), resultFor(2)]; + var clearedAll = false; + await pumpPanel( + tester, + result: history.last, + history: history, + onDelete: (_) {}, + onClearAll: () => clearedAll = true, + ); + + await openMenu(tester, Icons.delete_outline); + await tester.tap(find.text('Clear all results')); + await tester.pumpAndSettle(); + + expect(clearedAll, isTrue); + expect(find.text('2 result(s) cleared.'), findsOneWidget); + }); + + testWidgets('a run in flight cannot be deleted', (tester) async { + int? deleted; + await pumpPanel( + tester, + result: resultFor(2, status: 'Running · 2 workers', inFlight: true), + history: [resultFor(1)], + onDelete: (id) => deleted = id, + onClearAll: () {}, ); - expect(button.onPressed, isNull); + + await openMenu(tester, Icons.delete_outline); + await tester.tap(find.text('Delete current result')); + await tester.pumpAndSettle(); + + expect(deleted, isNull); + }); + + testWidgets('the copy and clear buttons are disabled before the first run', + (tester) async { + await pumpPanel(tester); + + for (final icon in [Icons.copy_all_outlined, Icons.delete_outline]) { + final button = tester.widget( + find.ancestor( + of: find.byIcon(icon), + matching: find.byType(IconButton), + ), + ); + expect(button.onPressed, isNull, reason: '$icon should be disabled'); + } + expect(find.text('Run a benchmark to see results.'), findsOneWidget); + expect(find.byType(DropdownButton), findsNothing); }); }