From 3af8ea9a446e820e08b1ca757840c51c052668d8 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sat, 27 Jun 2026 20:45:37 +0100 Subject: [PATCH 01/10] Add ANSI color to test output Status tokens are colored by outcome severity: green for ok, yellow for skipped/xfailed, magenta for XPASSED, bold-red for FAIL, red for ERROR. The summary line colors each category count the same way. Color is suppressed when stdout is not a TTY or NO_COLOR is set. --- src/testsweet/__main__.py | 22 ++++++++++++-- src/testsweet/_report.py | 57 ++++++++++++++++++++++++----------- tests/main.py | 63 +++++++++++++++++++++++++++++++++++++++ tests/report.py | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 19 deletions(-) create mode 100644 tests/main.py diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index cc3b9d9..68b2adc 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -1,4 +1,5 @@ import argparse +import os import pathlib import sys @@ -45,6 +46,22 @@ """ +def _supports_color() -> bool: + if not sys.stdout.isatty(): + return False + if os.environ.get('NO_COLOR'): + return False + if sys.platform == 'win32': + # VT processing is auto-enabled from Python 3.12+. On older + # versions, accept Windows Terminal (WT_SESSION) and ANSICON. + return ( + sys.version_info >= (3, 12) + or bool(os.environ.get('WT_SESSION')) + or bool(os.environ.get('ANSICON')) + ) + return True + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(add_help=False) parser.add_argument( @@ -94,6 +111,7 @@ def main(argv: list[str]) -> int: config = load_config(pathlib.Path.cwd()) plugins = load_plugins() wrap_unit = unit_wrapper(plugins) + use_color = _supports_color() results: list[tuple[str, Outcome]] = [] real_failures: list[tuple[str, Outcome]] = [] with session_for(plugins): @@ -106,13 +124,13 @@ def main(argv: list[str]) -> int: keep=keep, ): full_name = f'{module.__name__}.{name}' - print(format_result_line(full_name, outcome)) + print(format_result_line(full_name, outcome, use_color=use_color)) results.append((full_name, outcome)) if isinstance(outcome, (Failed, Errored, XPassed)): real_failures.append((full_name, outcome)) for full_name, outcome in real_failures: print_failure_detail(full_name, outcome) - print(summarize(results)) + print(summarize(results, use_color=use_color)) return 1 if real_failures else 0 diff --git a/src/testsweet/_report.py b/src/testsweet/_report.py index cd41fe5..6a2f586 100644 --- a/src/testsweet/_report.py +++ b/src/testsweet/_report.py @@ -19,6 +19,16 @@ XPassed, ) +_GREEN = '32' +_YELLOW = '33' +_RED = '31' +_BOLD_RED = '1;31' +_MAGENTA = '35' + + +def _c(text: str, code: str, enabled: bool) -> str: + return f'\033[{code}m{text}\033[0m' if enabled else text + _XPASS_DETAIL = ( 'Test was marked @xfail but passed. ' @@ -30,24 +40,34 @@ def _suffix(reason: str | None) -> str: return f': {reason}' if reason else '' -def format_result_line(full_name: str, outcome: Outcome) -> str: +def format_result_line( + full_name: str, + outcome: Outcome, + use_color: bool = False, +) -> str: """One-line summary suitable for streaming output.""" match outcome: case Passed(): - return f'{full_name} ... ok' + status = _c('ok', _GREEN, use_color) + return f'{full_name} ... {status}' case Skipped(reason=reason): - return f'{full_name} ... skipped{_suffix(reason)}' + status = _c(f'skipped{_suffix(reason)}', _YELLOW, use_color) + return f'{full_name} ... {status}' case XFailed(reason=reason): - return f'{full_name} ... xfailed{_suffix(reason)}' + status = _c(f'xfailed{_suffix(reason)}', _YELLOW, use_color) + return f'{full_name} ... {status}' case XPassed(reason=reason): - return f'{full_name} ... XPASSED{_suffix(reason)}' + status = _c(f'XPASSED{_suffix(reason)}', _MAGENTA, use_color) + return f'{full_name} ... {status}' case Failed(exc=exc): detail = str(exc) or assertion_source(exc) or '' - return f'{full_name} ... FAIL: AssertionError: {detail}' + status = _c(f'FAIL: AssertionError: {detail}', _BOLD_RED, use_color) + return f'{full_name} ... {status}' case Errored(exc=exc): - return ( - f'{full_name} ... ERROR: {type(exc).__name__}: {exc}' + status = _c( + f'ERROR: {type(exc).__name__}: {exc}', _RED, use_color, ) + return f'{full_name} ... {status}' def print_failure_detail( @@ -110,16 +130,19 @@ def _outcome_key(outcome: Outcome) -> str: _SUMMARY_ORDER = ( - ('passed', 'passed'), - ('failed', 'failed'), - ('errored', 'error'), - ('skipped', 'skipped'), - ('xfailed', 'xfailed'), - ('xpassed', 'xpassed'), + ('passed', 'passed', _GREEN), + ('failed', 'failed', _BOLD_RED), + ('errored', 'error', _RED), + ('skipped', 'skipped', _YELLOW), + ('xfailed', 'xfailed', _YELLOW), + ('xpassed', 'xpassed', _MAGENTA), ) -def summarize(results: Iterable[tuple[str, Outcome]]) -> str: +def summarize( + results: Iterable[tuple[str, Outcome]], + use_color: bool = False, +) -> str: """One-line summary of result counts.""" counts: collections.Counter = collections.Counter() total = 0 @@ -129,8 +152,8 @@ def summarize(results: Iterable[tuple[str, Outcome]]) -> str: if total == 0: return '0 tests' parts = [ - f'{counts[key]} {label}' - for key, label in _SUMMARY_ORDER + _c(f'{counts[key]} {label}', color, use_color) + for key, label, color in _SUMMARY_ORDER if counts.get(key) ] return ', '.join(parts) diff --git a/tests/main.py b/tests/main.py new file mode 100644 index 0000000..d4438eb --- /dev/null +++ b/tests/main.py @@ -0,0 +1,63 @@ +import os +from unittest.mock import patch + +from testsweet import test +from testsweet.__main__ import _supports_color + + +def _tty_env(**env_overrides): + """Return a patch.dict context that acts like a TTY with a clean env.""" + clean = {k: v for k, v in os.environ.items() + if k not in ('NO_COLOR', 'WT_SESSION', 'ANSICON')} + clean.update(env_overrides) + return patch.dict(os.environ, clean, clear=True) + + +@test +class SupportsColor: + def not_a_tty_returns_false(self): + with patch('sys.stdout') as m, _tty_env(): + m.isatty.return_value = False + assert not _supports_color() + + def no_color_env_returns_false(self): + with patch('sys.stdout') as m, _tty_env(NO_COLOR='1'): + m.isatty.return_value = True + assert not _supports_color() + + def non_windows_tty_returns_true(self): + with patch('sys.stdout') as m, patch('sys.platform', 'linux'), _tty_env(): + m.isatty.return_value = True + assert _supports_color() + + def windows_old_python_no_extras_returns_false(self): + with patch('sys.stdout') as m, \ + patch('sys.platform', 'win32'), \ + patch('sys.version_info', (3, 11)), \ + _tty_env(): + m.isatty.return_value = True + assert not _supports_color() + + def windows_python_312_returns_true(self): + with patch('sys.stdout') as m, \ + patch('sys.platform', 'win32'), \ + patch('sys.version_info', (3, 12)), \ + _tty_env(): + m.isatty.return_value = True + assert _supports_color() + + def windows_wt_session_returns_true(self): + with patch('sys.stdout') as m, \ + patch('sys.platform', 'win32'), \ + patch('sys.version_info', (3, 11)), \ + _tty_env(WT_SESSION='abc-123'): + m.isatty.return_value = True + assert _supports_color() + + def windows_ansicon_returns_true(self): + with patch('sys.stdout') as m, \ + patch('sys.platform', 'win32'), \ + patch('sys.version_info', (3, 11)), \ + _tty_env(ANSICON='80x24'): + m.isatty.return_value = True + assert _supports_color() diff --git a/tests/report.py b/tests/report.py index b143506..ba801b1 100644 --- a/tests/report.py +++ b/tests/report.py @@ -108,6 +108,58 @@ def errored_emits_error_block(self): assert 'TypeError' in out +@test +class FormatResultLineColor: + def pass_has_green_ok(self): + line = format_result_line('mod.t', Passed(), use_color=True) + assert '\x1b[32m' in line + assert 'ok' in line + + def fail_has_red_status(self): + line = format_result_line('mod.t', Failed(AssertionError('x')), use_color=True) + assert '\x1b[' in line + assert 'FAIL' in line + + def error_has_red_status(self): + line = format_result_line('mod.t', Errored(TypeError('x')), use_color=True) + assert '\x1b[' in line + assert 'ERROR' in line + + def skipped_has_yellow_status(self): + line = format_result_line('mod.t', Skipped(), use_color=True) + assert '\x1b[33m' in line + assert 'skipped' in line + + def xfailed_has_yellow_status(self): + line = format_result_line('mod.t', XFailed(ValueError('x')), use_color=True) + assert '\x1b[33m' in line + assert 'xfailed' in line + + def xpassed_has_magenta_status(self): + line = format_result_line('mod.t', XPassed(), use_color=True) + assert '\x1b[' in line + assert 'XPASSED' in line + + def no_color_by_default(self): + line = format_result_line('mod.t', Passed()) + assert '\x1b[' not in line + + +@test +class SummarizeColor: + def passed_count_is_green(self): + result = summarize([('a', Passed())], use_color=True) + assert '\x1b[32m' in result + + def failed_count_is_red(self): + result = summarize([('a', Failed(AssertionError()))], use_color=True) + assert '\x1b[1;31m' in result + + def no_color_by_default(self): + result = summarize([('a', Passed())]) + assert '\x1b[' not in result + + @test class Summarize: def empty_results(self): From 9cc1876e8cc56f7228c2bdba34855a6c42b6d291 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sat, 27 Jun 2026 22:28:10 +0100 Subject: [PATCH 02/10] Group test output by module, class, method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test output now uses a three-tier hierarchy: tests.catches ← module (unindented) CatchExceptions ← class (2-space indent) captures_exception... ← method (4-space indent) captures_subclass... CatchWarnings captures_single... Standalone functions appear at 2-space indent directly under the module header, with no intervening class line. --- src/testsweet/__main__.py | 23 +++++++++++- tests/cli.py | 75 +++++++++++++++++++++++++++++---------- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index 68b2adc..fd7398e 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -46,6 +46,15 @@ """ +def _split_group(name: str) -> tuple[str | None, str]: + """Split 'ClassName.method[n]' into ('ClassName', 'method[n]'). + + Returns (None, name) for standalone functions with no class prefix. + """ + dot = name.find('.') + return (name[:dot], name[dot + 1:]) if dot != -1 else (None, name) + + def _supports_color() -> bool: if not sys.stdout.isatty(): return False @@ -116,6 +125,8 @@ def main(argv: list[str]) -> int: real_failures: list[tuple[str, Outcome]] = [] with session_for(plugins): groups = discover_targets(args.targets, config) + last_module: str | None = None + last_class: str | None = None for module, names in groups: for name, outcome in run( module, @@ -124,7 +135,17 @@ def main(argv: list[str]) -> int: keep=keep, ): full_name = f'{module.__name__}.{name}' - print(format_result_line(full_name, outcome, use_color=use_color)) + class_name, short_name = _split_group(name) + if module.__name__ != last_module: + print(module.__name__) + last_module = module.__name__ + last_class = None + if class_name != last_class: + if class_name is not None: + print(f' {class_name}') + last_class = class_name + indent = ' ' if class_name else ' ' + print(f'{indent}{format_result_line(short_name, outcome, use_color=use_color)}') results.append((full_name, outcome)) if isinstance(outcome, (Failed, Errored, XPassed)): real_failures.append((full_name, outcome)) diff --git a/tests/cli.py b/tests/cli.py index d44e8c1..f6b704f 100644 --- a/tests/cli.py +++ b/tests/cli.py @@ -40,7 +40,7 @@ def {func_name}(): ), ( ('tests.fixtures.runner.class_simple',), - ['Simple.first ... ok', 'Simple.second ... ok'], + [' first ... ok', ' second ... ok'], ), ( ('tests.fixtures.runner.params_simple',), @@ -85,8 +85,8 @@ def selector_argv_runs_one_method(self): 'tests.fixtures.runner.class_simple.Simple.first', ) assert result.returncode == 0 - assert 'Simple.first ... ok' in result.stdout - assert 'Simple.second' not in result.stdout + assert ' first ... ok' in result.stdout + assert 'second' not in result.stdout def two_module_targets(self): result = _run_cli( @@ -104,8 +104,8 @@ def two_selectors_same_module_grouped(self): ) assert result.returncode == 0 # Both methods, single grouped run — neither line repeats. - assert result.stdout.count('Simple.first ... ok') == 1 - assert result.stdout.count('Simple.second ... ok') == 1 + assert result.stdout.count(' first ... ok') == 1 + assert result.stdout.count(' second ... ok') == 1 def module_target_overrides_selector_for_same_module(self): result = _run_cli( @@ -468,9 +468,9 @@ def include_tag_runs_only_matching(self): assert result.returncode == 0 # Class-level @tag('slow') on SlowSuite propagates to both # methods; lone_function carries @tag('slow') directly. - assert 'SlowSuite.alpha ... ok' in result.stdout - assert 'SlowSuite.beta ... ok' in result.stdout - assert 'lone_function ... ok' in result.stdout + assert ' alpha ... ok' in result.stdout + assert ' beta ... ok' in result.stdout + assert ' lone_function ... ok' in result.stdout assert 'untagged_function' not in result.stdout assert 'Untagged.delta' not in result.stdout @@ -480,9 +480,9 @@ def exclude_tag_drops_matching(self): 'tests.fixtures.runner.tagged_class', ) assert result.returncode == 0 - assert 'untagged_function ... ok' in result.stdout - assert 'Untagged.delta ... ok' in result.stdout - assert 'SlowSuite.alpha' not in result.stdout + assert ' untagged_function ... ok' in result.stdout + assert ' delta ... ok' in result.stdout + assert 'tagged_class.SlowSuite' not in result.stdout assert 'lone_function' not in result.stdout def long_form_flags_work(self): @@ -492,9 +492,9 @@ def long_form_flags_work(self): 'tests.fixtures.runner.tagged_class', ) assert result.returncode == 0 - assert 'Untagged.gamma ... ok' in result.stdout + assert ' gamma ... ok' in result.stdout # SlowSuite.beta has db but also slow (from class) — vetoed. - assert 'SlowSuite.beta' not in result.stdout + assert 'tagged_class.SlowSuite' not in result.stdout def repeated_include_is_or(self): result = _run_cli( @@ -506,8 +506,8 @@ def repeated_include_is_or(self): # Untagged.delta drop out. assert 'untagged_function' not in result.stdout assert 'Untagged.delta' not in result.stdout - assert 'SlowSuite.alpha ... ok' in result.stdout - assert 'Untagged.gamma ... ok' in result.stdout + assert ' alpha ... ok' in result.stdout + assert ' gamma ... ok' in result.stdout def overlapping_include_and_exclude_errors(self): result = _run_cli('-t', 'slow', '-T', 'slow') @@ -518,13 +518,50 @@ def overlapping_include_and_exclude_errors(self): def empty_filter_runs_everything(self): result = _run_cli('tests.fixtures.runner.tagged_class') assert result.returncode == 0 - assert 'untagged_function ... ok' in result.stdout - assert 'SlowSuite.alpha ... ok' in result.stdout + assert ' untagged_function ... ok' in result.stdout + assert ' alpha ... ok' in result.stdout def non_assertion_error_has_no_explanation_block(self): result = _run_cli('tests.fixtures.runner.non_assertion_error') assert result.returncode == 1 lines = result.stdout.splitlines() - # Trailing line is the summary; the traceback line is just before it. assert lines[-1] == '1 error' - assert lines[-2] == 'ValueError: boom' + assert 'ValueError: boom' in result.stdout + + def module_header_printed_once_per_module(self): + result = _run_cli('tests.fixtures.runner.all_pass') + assert result.returncode == 0 + lines = result.stdout.splitlines() + header_lines = [l for l in lines if l == 'tests.fixtures.runner.all_pass'] + assert len(header_lines) == 1 + + def result_lines_are_indented(self): + result = _run_cli('tests.fixtures.runner.all_pass') + assert result.returncode == 0 + assert ' passes_one ... ok' in result.stdout + assert ' passes_two ... ok' in result.stdout + + def class_header_indented_under_module(self): + result = _run_cli('tests.fixtures.runner.class_simple') + assert result.returncode == 0 + lines = result.stdout.splitlines() + # Module line unindented, class line 2-space indented + assert 'tests.fixtures.runner.class_simple' in lines + assert ' Simple' in lines + + def class_method_result_line_four_space_indent(self): + result = _run_cli('tests.fixtures.runner.class_simple') + assert result.returncode == 0 + assert ' first ... ok' in result.stdout + assert ' second ... ok' in result.stdout + # The class prefix must not appear on result lines + assert 'Simple.first' not in result.stdout + assert 'Simple.second' not in result.stdout + + def two_modules_each_get_header(self): + result = _run_cli( + 'tests.fixtures.runner.all_pass', + 'tests.fixtures.runner.has_failure', + ) + assert 'tests.fixtures.runner.all_pass' in result.stdout + assert 'tests.fixtures.runner.has_failure' in result.stdout From d026d13531b2d2618eaf127995c74ddc4283e01e Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sat, 27 Jun 2026 22:56:06 +0100 Subject: [PATCH 03/10] Add elapsed time to the summary line The summary (e.g. '297 passed') now shows how long the run took, e.g. '297 passed in 1.23s'. The elapsed parameter is optional so the summarize() function stays usable without timing context. --- src/testsweet/__main__.py | 5 ++++- src/testsweet/_report.py | 6 ++++-- tests/cli.py | 9 ++++++++- tests/report.py | 20 ++++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index fd7398e..4e682db 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -2,6 +2,7 @@ import os import pathlib import sys +import time from testsweet._config import load_config from testsweet._loaders import scoped_sys_path @@ -123,6 +124,7 @@ def main(argv: list[str]) -> int: use_color = _supports_color() results: list[tuple[str, Outcome]] = [] real_failures: list[tuple[str, Outcome]] = [] + start = time.monotonic() with session_for(plugins): groups = discover_targets(args.targets, config) last_module: str | None = None @@ -149,9 +151,10 @@ def main(argv: list[str]) -> int: results.append((full_name, outcome)) if isinstance(outcome, (Failed, Errored, XPassed)): real_failures.append((full_name, outcome)) + elapsed = time.monotonic() - start for full_name, outcome in real_failures: print_failure_detail(full_name, outcome) - print(summarize(results, use_color=use_color)) + print(summarize(results, use_color=use_color, elapsed=elapsed)) return 1 if real_failures else 0 diff --git a/src/testsweet/_report.py b/src/testsweet/_report.py index 6a2f586..9d4ed36 100644 --- a/src/testsweet/_report.py +++ b/src/testsweet/_report.py @@ -142,6 +142,7 @@ def _outcome_key(outcome: Outcome) -> str: def summarize( results: Iterable[tuple[str, Outcome]], use_color: bool = False, + elapsed: float | None = None, ) -> str: """One-line summary of result counts.""" counts: collections.Counter = collections.Counter() @@ -149,11 +150,12 @@ def summarize( for _name, outcome in results: counts[_outcome_key(outcome)] += 1 total += 1 + timing = f' in {elapsed:.2f}s' if elapsed is not None else '' if total == 0: - return '0 tests' + return f'0 tests{timing}' parts = [ _c(f'{counts[key]} {label}', color, use_color) for key, label, color in _SUMMARY_ORDER if counts.get(key) ] - return ', '.join(parts) + return f'{", ".join(parts)}{timing}' diff --git a/tests/cli.py b/tests/cli.py index f6b704f..59958c6 100644 --- a/tests/cli.py +++ b/tests/cli.py @@ -525,7 +525,7 @@ def non_assertion_error_has_no_explanation_block(self): result = _run_cli('tests.fixtures.runner.non_assertion_error') assert result.returncode == 1 lines = result.stdout.splitlines() - assert lines[-1] == '1 error' + assert lines[-1].startswith('1 error') assert 'ValueError: boom' in result.stdout def module_header_printed_once_per_module(self): @@ -558,6 +558,13 @@ def class_method_result_line_four_space_indent(self): assert 'Simple.first' not in result.stdout assert 'Simple.second' not in result.stdout + def summary_includes_timing(self): + result = _run_cli('tests.fixtures.runner.all_pass') + assert result.returncode == 0 + lines = result.stdout.splitlines() + assert lines[-1].startswith('2 passed in ') + assert lines[-1].endswith('s') + def two_modules_each_get_header(self): result = _run_cli( 'tests.fixtures.runner.all_pass', diff --git a/tests/report.py b/tests/report.py index ba801b1..8e1ec9d 100644 --- a/tests/report.py +++ b/tests/report.py @@ -160,6 +160,26 @@ def no_color_by_default(self): assert '\x1b[' not in result +@test +class SummarizeTiming: + def elapsed_appended_to_summary(self): + results = [('a', Passed())] + out = summarize(results, elapsed=1.5) + assert out == '1 passed in 1.50s' + + def elapsed_zero(self): + out = summarize([('a', Passed())], elapsed=0.0) + assert out == '1 passed in 0.00s' + + def no_elapsed_omits_timing(self): + out = summarize([('a', Passed())]) + assert 'in' not in out + + def elapsed_on_empty_results(self): + out = summarize([], elapsed=0.5) + assert out == '0 tests in 0.50s' + + @test class Summarize: def empty_results(self): From 29b528820a2b83477d17f84e6e96c8e11429f49d Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sat, 27 Jun 2026 22:56:38 +0100 Subject: [PATCH 04/10] Print blank line before summary Visually separates the per-test result lines (and any failure detail blocks) from the final summary count, making it easier to spot the verdict at a glance. --- src/testsweet/__main__.py | 1 + tests/cli.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index 4e682db..6144545 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -154,6 +154,7 @@ def main(argv: list[str]) -> int: elapsed = time.monotonic() - start for full_name, outcome in real_failures: print_failure_detail(full_name, outcome) + print() print(summarize(results, use_color=use_color, elapsed=elapsed)) return 1 if real_failures else 0 diff --git a/tests/cli.py b/tests/cli.py index 59958c6..6dfbeaa 100644 --- a/tests/cli.py +++ b/tests/cli.py @@ -558,6 +558,13 @@ def class_method_result_line_four_space_indent(self): assert 'Simple.first' not in result.stdout assert 'Simple.second' not in result.stdout + def blank_line_before_summary(self): + result = _run_cli('tests.fixtures.runner.all_pass') + assert result.returncode == 0 + lines = result.stdout.splitlines() + # Summary is last; blank separator is second-to-last. + assert lines[-2] == '' + def summary_includes_timing(self): result = _run_cli('tests.fixtures.runner.all_pass') assert result.returncode == 0 From 4cd036abf4ae0afcbce0bc7ce4e5f76f00a14912 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sun, 28 Jun 2026 00:50:13 +0100 Subject: [PATCH 05/10] Add Result dataclass carrying captured output --- src/testsweet/__init__.py | 2 ++ src/testsweet/_outcomes.py | 16 ++++++++++++++++ tests/outcomes.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/src/testsweet/__init__.py b/src/testsweet/__init__.py index 7de962a..91542ce 100644 --- a/src/testsweet/__init__.py +++ b/src/testsweet/__init__.py @@ -7,6 +7,7 @@ Failed, Outcome, Passed, + Result, Skipped, XFailed, XPassed, @@ -27,6 +28,7 @@ 'Outcome', 'Passed', 'Plugin', + 'Result', 'Skipped', 'XFailed', 'XPassed', diff --git a/src/testsweet/_outcomes.py b/src/testsweet/_outcomes.py index d501663..964ff13 100644 --- a/src/testsweet/_outcomes.py +++ b/src/testsweet/_outcomes.py @@ -59,3 +59,19 @@ class XPassed: Outcome = Passed | Failed | Errored | Skipped | XFailed | XPassed + + +@dataclass(frozen=True) +class Result: + """A test unit's outcome plus any output it produced. + + ``run()`` returns ``list[Result]``. ``stdout`` and ``stderr`` hold + the test's captured streams; they are empty unless the unit wrote + to those streams. Captured output is replayed only when the test + fails — see ``_report.print_failure_detail``. + """ + + name: str + outcome: Outcome + stdout: str = '' + stderr: str = '' diff --git a/tests/outcomes.py b/tests/outcomes.py index 5ed0a21..bcc47d1 100644 --- a/tests/outcomes.py +++ b/tests/outcomes.py @@ -4,6 +4,7 @@ Errored, Failed, Passed, + Result, Skipped, XFailed, XPassed, @@ -90,3 +91,31 @@ def reason_defaults_to_none(self): def is_not_an_exception(self): assert not isinstance(XPassed(), Exception) + + +@test +class ResultRecord: + def carries_name_and_outcome(self): + r = Result('mod.t', Passed()) + assert r.name == 'mod.t' + assert isinstance(r.outcome, Passed) + + def capture_defaults_empty(self): + r = Result('mod.t', Passed()) + assert r.stdout == '' + assert r.stderr == '' + + def carries_captured_streams(self): + r = Result('mod.t', Passed(), stdout='hello\n', stderr='oops\n') + assert r.stdout == 'hello\n' + assert r.stderr == 'oops\n' + + def is_frozen(self): + import dataclasses + r = Result('mod.t', Passed()) + try: + r.name = 'other' # ty: ignore[invalid-assignment] + except dataclasses.FrozenInstanceError: + pass + else: + raise AssertionError('Result should be frozen') From d8de061844729778a825d7d2f1235b79cc2ebd4e Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sun, 28 Jun 2026 01:17:12 +0100 Subject: [PATCH 06/10] Capture stdout/stderr per unit and return Result records --- src/testsweet/__main__.py | 18 +-- src/testsweet/_report.py | 7 +- src/testsweet/_runner.py | 59 ++++++--- tests/report.py | 44 ++++--- tests/runner.py | 260 +++++++++++++++++++++++++------------- 5 files changed, 247 insertions(+), 141 deletions(-) diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index 6144545..32b893c 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -7,7 +7,7 @@ from testsweet._config import load_config from testsweet._loaders import scoped_sys_path from testsweet._plugins import load_plugins, session_for, unit_wrapper -from testsweet._outcomes import Errored, Failed, Outcome, XPassed +from testsweet._outcomes import Errored, Failed, Outcome, Result, XPassed from testsweet._report import ( format_result_line, print_failure_detail, @@ -122,7 +122,7 @@ def main(argv: list[str]) -> int: plugins = load_plugins() wrap_unit = unit_wrapper(plugins) use_color = _supports_color() - results: list[tuple[str, Outcome]] = [] + results: list[Result] = [] real_failures: list[tuple[str, Outcome]] = [] start = time.monotonic() with session_for(plugins): @@ -130,14 +130,14 @@ def main(argv: list[str]) -> int: last_module: str | None = None last_class: str | None = None for module, names in groups: - for name, outcome in run( + for result in run( module, names=names, wrap_unit=wrap_unit, keep=keep, ): - full_name = f'{module.__name__}.{name}' - class_name, short_name = _split_group(name) + full_name = f'{module.__name__}.{result.name}' + class_name, short_name = _split_group(result.name) if module.__name__ != last_module: print(module.__name__) last_module = module.__name__ @@ -147,10 +147,10 @@ def main(argv: list[str]) -> int: print(f' {class_name}') last_class = class_name indent = ' ' if class_name else ' ' - print(f'{indent}{format_result_line(short_name, outcome, use_color=use_color)}') - results.append((full_name, outcome)) - if isinstance(outcome, (Failed, Errored, XPassed)): - real_failures.append((full_name, outcome)) + print(f'{indent}{format_result_line(short_name, result.outcome, use_color=use_color)}') + results.append(result) + if isinstance(result.outcome, (Failed, Errored, XPassed)): + real_failures.append((full_name, result.outcome)) elapsed = time.monotonic() - start for full_name, outcome in real_failures: print_failure_detail(full_name, outcome) diff --git a/src/testsweet/_report.py b/src/testsweet/_report.py index 9d4ed36..b9fc568 100644 --- a/src/testsweet/_report.py +++ b/src/testsweet/_report.py @@ -14,6 +14,7 @@ Failed, Outcome, Passed, + Result, Skipped, XFailed, XPassed, @@ -140,15 +141,15 @@ def _outcome_key(outcome: Outcome) -> str: def summarize( - results: Iterable[tuple[str, Outcome]], + results: Iterable[Result], use_color: bool = False, elapsed: float | None = None, ) -> str: """One-line summary of result counts.""" counts: collections.Counter = collections.Counter() total = 0 - for _name, outcome in results: - counts[_outcome_key(outcome)] += 1 + for r in results: + counts[_outcome_key(r.outcome)] += 1 total += 1 timing = f' in {elapsed:.2f}s' if elapsed is not None else '' if total == 0: diff --git a/src/testsweet/_runner.py b/src/testsweet/_runner.py index 3f5c3a6..15f685a 100644 --- a/src/testsweet/_runner.py +++ b/src/testsweet/_runner.py @@ -1,5 +1,11 @@ """Run resolved test units and collect results.""" -from contextlib import AbstractContextManager, nullcontext +import io +from contextlib import ( + AbstractContextManager, + nullcontext, + redirect_stderr, + redirect_stdout, +) from types import ModuleType from typing import Any, Callable @@ -10,6 +16,7 @@ Failed, Outcome, Passed, + Result, Skipped, XFailed, XPassed, @@ -24,7 +31,7 @@ def run( names: list[str] | None = None, wrap_unit: Callable[[str], AbstractContextManager[Any]] | None = None, keep: TagFilter | None = None, -) -> list[tuple[str, Outcome]]: +) -> list[Result]: """Run the tests in ``module``. If ``names`` is given, only run tests whose qualified names appear @@ -35,16 +42,18 @@ def run( method's effective tag set is the union of its class's tags and its own. - Returns a list of ``(name, outcome)`` tuples. ``outcome`` is one - of ``Passed``, ``Failed``, ``Errored``, ``Skipped``, ``XFailed``, - ``XPassed``. + Returns a list of ``Result`` records. Each carries the unit's + ``name``, its ``outcome`` (one of ``Passed``, ``Failed``, + ``Errored``, ``Skipped``, ``XFailed``, ``XPassed``), and the + ``stdout``/``stderr`` it printed (empty unless it wrote to those + streams; captured only while the unit body runs). """ if wrap_unit is None: def wrap_unit(_name: str) -> AbstractContextManager[Any]: return nullcontext() - results: list[tuple[str, Outcome]] = [] + results: list[Result] = [] for name, call in resolve_units(module, names, keep=keep): - results.append((name, _run_one(name, call, wrap_unit))) + results.append(_run_one(name, call, wrap_unit)) return results @@ -52,29 +61,43 @@ def _run_one( name: str, call: Callable[[], Any], wrap_unit: Callable[[str], AbstractContextManager[Any]], -) -> Outcome: +) -> Result: try: skip_marker: SkipMarker | None = active_marker(call, SKIP_MARKER) except Exception as exc: - return Errored(exc) + return Result(name, Errored(exc)) if skip_marker is not None: - return Skipped(skip_marker.reason) + return Result(name, Skipped(skip_marker.reason)) try: xfail_marker: XFailMarker | None = active_marker(call, XFAIL_MARKER) except Exception as exc: - return Errored(exc) + return Result(name, Errored(exc)) + out_buf = io.StringIO() + err_buf = io.StringIO() if xfail_marker is not None: try: - with wrap_unit(name): + with ( + wrap_unit(name), + redirect_stdout(out_buf), + redirect_stderr(err_buf), + ): call() except Exception as exc: - return XFailed(exc, xfail_marker.reason) - return XPassed(xfail_marker.reason) + outcome: Outcome = XFailed(exc, xfail_marker.reason) + else: + outcome = XPassed(xfail_marker.reason) + return Result(name, outcome, out_buf.getvalue(), err_buf.getvalue()) try: - with wrap_unit(name): + with ( + wrap_unit(name), + redirect_stdout(out_buf), + redirect_stderr(err_buf), + ): call() except AssertionError as exc: - return Failed(exc) + outcome = Failed(exc) except Exception as exc: - return Errored(exc) - return Passed() + outcome = Errored(exc) + else: + outcome = Passed() + return Result(name, outcome, out_buf.getvalue(), err_buf.getvalue()) diff --git a/tests/report.py b/tests/report.py index 8e1ec9d..fb63a05 100644 --- a/tests/report.py +++ b/tests/report.py @@ -5,6 +5,7 @@ Errored, Failed, Passed, + Result, Skipped, XFailed, XPassed, @@ -16,7 +17,7 @@ ) Outcome = Passed | Failed | Errored | Skipped | XFailed | XPassed -Results = list[tuple[str, Outcome]] +Results = list[Result] def _capture(full_name, outcome): @@ -148,31 +149,34 @@ def no_color_by_default(self): @test class SummarizeColor: def passed_count_is_green(self): - result = summarize([('a', Passed())], use_color=True) + result = summarize([Result('a', Passed())], use_color=True) assert '\x1b[32m' in result def failed_count_is_red(self): - result = summarize([('a', Failed(AssertionError()))], use_color=True) + result = summarize( + [Result('a', Failed(AssertionError()))], + use_color=True, + ) assert '\x1b[1;31m' in result def no_color_by_default(self): - result = summarize([('a', Passed())]) + result = summarize([Result('a', Passed())]) assert '\x1b[' not in result @test class SummarizeTiming: def elapsed_appended_to_summary(self): - results = [('a', Passed())] + results = [Result('a', Passed())] out = summarize(results, elapsed=1.5) assert out == '1 passed in 1.50s' def elapsed_zero(self): - out = summarize([('a', Passed())], elapsed=0.0) + out = summarize([Result('a', Passed())], elapsed=0.0) assert out == '1 passed in 0.00s' def no_elapsed_omits_timing(self): - out = summarize([('a', Passed())]) + out = summarize([Result('a', Passed())]) assert 'in' not in out def elapsed_on_empty_results(self): @@ -186,18 +190,18 @@ def empty_results(self): assert summarize([]) == '0 tests' def all_passes(self): - results = [('a', Passed()), ('b', Passed())] + results = [Result('a', Passed()), Result('b', Passed())] assert summarize(results) == '2 passed' def mixed_outcomes(self): results: Results = [ - ('p1', Passed()), - ('p2', Passed()), - ('f1', Failed(AssertionError('x'))), - ('e1', Errored(TypeError('y'))), - ('s1', Skipped('later')), - ('xf1', XFailed(ValueError('z'))), - ('xp1', XPassed()), + Result('p1', Passed()), + Result('p2', Passed()), + Result('f1', Failed(AssertionError('x'))), + Result('e1', Errored(TypeError('y'))), + Result('s1', Skipped('later')), + Result('xf1', XFailed(ValueError('z'))), + Result('xp1', XPassed()), ] assert summarize(results) == ( '2 passed, 1 failed, 1 error, ' @@ -205,7 +209,7 @@ def mixed_outcomes(self): ) def zero_categories_omitted(self): - results = [('s1', Skipped()), ('s2', Skipped())] + results = [Result('s1', Skipped()), Result('s2', Skipped())] assert summarize(results) == '2 skipped' def category_order_is_stable(self): @@ -213,10 +217,10 @@ def category_order_is_stable(self): # always emits passed → failed → error → skipped → xfailed # → xpassed. results: Results = [ - ('xp1', XPassed()), - ('p1', Passed()), - ('s1', Skipped()), - ('f1', Failed(AssertionError())), + Result('xp1', XPassed()), + Result('p1', Passed()), + Result('s1', Skipped()), + Result('f1', Failed(AssertionError())), ] assert summarize(results) == ( '1 passed, 1 failed, 1 skipped, 1 xpassed' diff --git a/tests/runner.py b/tests/runner.py index f96c334..a0413c9 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -24,26 +24,26 @@ def single_passing_test(self): ) results = run(mod) assert len(results) == 2 - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) def single_failing_assert(self): mod = importlib.import_module( 'tests.fixtures.runner.has_failure', ) results = run(mod) - assert results[0][0] == 'passes' - assert isinstance(results[0][1], Passed) - assert results[1][0] == 'fails' - assert isinstance(results[1][1], Failed) - assert isinstance(results[1][1].exc, AssertionError) + assert results[0].name == 'passes' + assert isinstance(results[0].outcome, Passed) + assert results[1].name == 'fails' + assert isinstance(results[1].outcome, Failed) + assert isinstance(results[1].outcome.exc, AssertionError) def results_in_discover_order(self): mod = importlib.import_module( 'tests.fixtures.runner.has_failure', ) results = run(mod) - assert [name for name, _ in results] == ['passes', 'fails'] + assert [r.name for r in results] == ['passes', 'fails'] def empty_module_returns_empty_list(self): mod = importlib.import_module( @@ -58,10 +58,9 @@ def non_assertion_exception_is_caught(self): ) results = run(mod) assert len(results) == 1 - name, outcome = results[0] - assert name == 'raises_value_error' - assert isinstance(outcome, Errored) - assert isinstance(outcome.exc, ValueError) + assert results[0].name == 'raises_value_error' + assert isinstance(results[0].outcome, Errored) + assert isinstance(results[0].outcome.exc, ValueError) def keyboard_interrupt_propagates(self): mod = importlib.import_module( @@ -83,17 +82,17 @@ def class_with_passing_methods(self): ) results = run(mod) assert len(results) == 2 - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['Simple.first', 'Simple.second'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) def underscore_methods_are_skipped(self): mod = importlib.import_module( 'tests.fixtures.runner.class_with_underscore_methods', ) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['WithUnderscores.public'] def enter_and_exit_run_around_methods(self): @@ -110,11 +109,11 @@ def failing_method_does_not_abort_class(self): ) results = run(mod) assert len(results) == 2 - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['HasFailure.passes', 'HasFailure.fails'] - assert isinstance(results[0][1], Passed) - assert isinstance(results[1][1], Failed) - assert isinstance(results[1][1].exc, AssertionError) + assert isinstance(results[0].outcome, Passed) + assert isinstance(results[1].outcome, Failed) + assert isinstance(results[1].outcome.exc, AssertionError) def enter_exception_propagates(self): mod = importlib.import_module( @@ -139,7 +138,7 @@ def mixed_function_and_class_in_vars_order(self): 'tests.fixtures.runner.class_mixed_with_function', ) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['free_function', 'ClassUnit.method'] @@ -150,22 +149,22 @@ def runs_each_tuple_in_order(self): 'tests.fixtures.runner.params_simple', ) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['adds[0]', 'adds[1]'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) def failure_recorded_at_correct_index(self): mod = importlib.import_module( 'tests.fixtures.runner.params_with_failure', ) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['adds[0]', 'adds[1]', 'adds[2]'] - assert isinstance(results[0][1], Passed) - assert isinstance(results[1][1], Failed) - assert isinstance(results[1][1].exc, AssertionError) - assert isinstance(results[2][1], Passed) + assert isinstance(results[0].outcome, Passed) + assert isinstance(results[1].outcome, Failed) + assert isinstance(results[1].outcome.exc, AssertionError) + assert isinstance(results[2].outcome, Passed) def empty_param_list_produces_no_results(self): mod = importlib.import_module( @@ -178,10 +177,10 @@ def function_without_params_unchanged(self): 'tests.fixtures.runner.params_no_decoration', ) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['plain', 'parameterized[0]'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) def accepts_generator(self): # The generator was consumed at decoration time, so the second @@ -191,12 +190,12 @@ def accepts_generator(self): ) first = run(mod) second = run(mod) - assert [name for name, _ in first] == [ + assert [r.name for r in first] == [ 'adds[0]', 'adds[1]', 'adds[2]', ] - assert [name for name, _ in second] == [ + assert [r.name for r in second] == [ 'adds[0]', 'adds[1]', 'adds[2]', @@ -207,10 +206,10 @@ def on_class_method(self): 'tests.fixtures.runner.params_on_class_method', ) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['Cls.method[0]', 'Cls.method[1]'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) @test @@ -222,10 +221,10 @@ def runs_each_yielded_tuple(self): # Re-import so the module-level generator is freshly created. importlib.reload(mod) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['adds[0]', 'adds[1]', 'adds[2]'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) def generator_is_consumed_after_first_run(self): mod = importlib.import_module( @@ -244,8 +243,8 @@ def list_is_idempotent(self): importlib.reload(mod) first = run(mod) second = run(mod) - names_first = [name for name, _ in first] - names_second = [name for name, _ in second] + names_first = [r.name for r in first] + names_second = [r.name for r in second] assert names_first == ['equals[0]', 'equals[1]'] assert names_second == ['equals[0]', 'equals[1]'] @@ -255,10 +254,10 @@ def on_class_method(self): ) importlib.reload(mod) results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['Cls.method[0]', 'Cls.method[1]'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) @test @@ -268,14 +267,14 @@ def filters_to_named_function(self): 'tests.fixtures.runner.all_pass', ) results = run(mod, names=['passes_one']) - assert [name for name, _ in results] == ['passes_one'] + assert [r.name for r in results] == ['passes_one'] def class_name_runs_all_methods(self): mod = importlib.import_module( 'tests.fixtures.runner.class_simple', ) results = run(mod, names=['Simple']) - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'Simple.first', 'Simple.second', ] @@ -285,7 +284,7 @@ def class_method_selector_runs_one(self): 'tests.fixtures.runner.class_simple', ) results = run(mod, names=['Simple.first']) - assert [name for name, _ in results] == ['Simple.first'] + assert [r.name for r in results] == ['Simple.first'] def two_method_selectors_run_in_vars_order(self): mod = importlib.import_module( @@ -296,7 +295,7 @@ def two_method_selectors_run_in_vars_order(self): names=['Simple.second', 'Simple.first'], ) # vars() order, NOT argv order — Simple.first defined first. - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'Simple.first', 'Simple.second', ] @@ -306,7 +305,7 @@ def class_form_wins_over_method_form(self): 'tests.fixtures.runner.class_simple', ) results = run(mod, names=['Simple', 'Simple.first']) - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'Simple.first', 'Simple.second', ] @@ -339,7 +338,7 @@ def parameterized_function_selector_runs_all_params(self): 'tests.fixtures.runner.params_simple', ) results = run(mod, names=['adds']) - assert [name for name, _ in results] == ['adds[0]', 'adds[1]'] + assert [r.name for r in results] == ['adds[0]', 'adds[1]'] def class_method_unknown_method_raises(self): mod = importlib.import_module( @@ -358,9 +357,9 @@ def runs_decorated_class_without_context_manager(self): 'tests.fixtures.runner.class_decorated_simple', ) results = run(mod) - names = sorted(name for name, _ in results) + names = sorted(r.name for r in results) assert names == ['Simple.fails', 'Simple.passes'] - outcomes = dict(results) + outcomes = {r.name: r.outcome for r in results} assert isinstance(outcomes['Simple.passes'], Passed) assert isinstance(outcomes['Simple.fails'], Failed) assert isinstance(outcomes['Simple.fails'].exc, AssertionError) @@ -370,8 +369,8 @@ def runs_decorated_class_with_context_manager(self): 'tests.fixtures.runner.class_decorated_with_cm', ) results = run(mod) - assert [name for name, _ in results] == ['WithCM.uses_fixture'] - assert isinstance(results[0][1], Passed) + assert [r.name for r in results] == ['WithCM.uses_fixture'] + assert isinstance(results[0].outcome, Passed) assert mod.CALLS == ['enter', 'test', 'exit'] def class_with_enter_only_propagates_type_error(self): @@ -532,7 +531,7 @@ def wrap(name): 'tests.fixtures.runner.all_pass', ) results = run(mod, wrap_unit=wrap) - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'passes_one', 'passes_two', ] assert events == [ @@ -557,9 +556,9 @@ def wrap(name): 'tests.fixtures.runner.all_pass', ) results = run(mod, wrap_unit=wrap) - for _, outcome in results: - assert isinstance(outcome, Errored) - assert isinstance(outcome.exc, RuntimeError) + for r in results: + assert isinstance(r.outcome, Errored) + assert isinstance(r.outcome.exc, RuntimeError) def wrap_unit_exit_failure_attributed_to_test(self): @contextmanager @@ -573,9 +572,9 @@ def wrap(name): 'tests.fixtures.runner.all_pass', ) results = run(mod, wrap_unit=wrap) - for _, outcome in results: - assert isinstance(outcome, Errored) - assert isinstance(outcome.exc, RuntimeError) + for r in results: + assert isinstance(r.outcome, Errored) + assert isinstance(r.outcome.exc, RuntimeError) @test @@ -588,10 +587,10 @@ def params_combine_with_test_context(self): ) mod.CALLS.clear() results = run(mod) - names = [name for name, _ in results] + names = [r.name for r in results] assert names == ['Cls.method[0]', 'Cls.method[1]'] - for _, outcome in results: - assert isinstance(outcome, Passed) + for r in results: + assert isinstance(r.outcome, Passed) assert mod.CALLS == [ 'enter', 'ctx-enter', 'method(1,2)', 'ctx-exit', @@ -604,10 +603,10 @@ def test_context_enter_failure_attributed_to_test(self): 'tests.fixtures.runner.class_test_context_raises', ) results = run(mod, names=['TestContextEnterRaises']) - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'TestContextEnterRaises.passes', ] - outcome = results[0][1] + outcome = results[0].outcome assert isinstance(outcome, Errored) assert isinstance(outcome.exc, RuntimeError) assert 'enter failed' in str(outcome.exc) @@ -617,10 +616,10 @@ def test_context_exit_failure_attributed_to_test(self): 'tests.fixtures.runner.class_test_context_raises', ) results = run(mod, names=['TestContextExitRaises']) - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'TestContextExitRaises.passes', ] - outcome = results[0][1] + outcome = results[0].outcome assert isinstance(outcome, Errored) assert isinstance(outcome.exc, RuntimeError) assert 'exit failed' in str(outcome.exc) @@ -642,7 +641,7 @@ def runs(): mod = _module_with(skipped=skipped, runs=runs) results = run(mod) - outcomes = dict(results) + outcomes = {r.name: r.outcome for r in results} assert isinstance(outcomes['skipped'], Skipped) assert isinstance(outcomes['runs'], Passed) assert called == ['runs'] @@ -659,8 +658,8 @@ def maybe(): results = run(mod) assert called == ['ran'] assert len(results) == 1 - assert results[0][0] == 'maybe' - assert isinstance(results[0][1], Passed) + assert results[0].name == 'maybe' + assert isinstance(results[0].outcome, Passed) def skip_with_callable_condition_evaluated_at_run_time(self): # The callable is invoked when the runner gets to the test, @@ -680,7 +679,7 @@ def maybe(): mod = _module_with(maybe=maybe) results = run(mod) assert called == ['cond'] - outcome = results[0][1] + outcome = results[0].outcome assert isinstance(outcome, Skipped) assert outcome.reason == 'lazy' @@ -695,7 +694,7 @@ def maybe(): mod = _module_with(maybe=maybe) results = run(mod) assert called == ['ran'] - assert isinstance(results[0][1], Passed) + assert isinstance(results[0].outcome, Passed) def skip_callable_condition_raising_records_errored(self): @test @@ -705,7 +704,7 @@ def maybe(): mod = _module_with(maybe=maybe) results = run(mod) - outcome = results[0][1] + outcome = results[0].outcome assert isinstance(outcome, Errored) assert isinstance(outcome.exc, ZeroDivisionError) @@ -717,7 +716,7 @@ def pending(): mod = _module_with(pending=pending) results = run(mod) - _, exc = results[0] + exc = results[0].outcome assert isinstance(exc, Skipped) assert exc.reason == 'not yet implemented' @@ -729,7 +728,7 @@ def broken(): mod = _module_with(broken=broken) results = run(mod) - _, exc = results[0] + exc = results[0].outcome assert isinstance(exc, XFailed) assert isinstance(exc.actual, ValueError) assert exc.reason == 'known bug' @@ -742,7 +741,7 @@ def secretly_works(): mod = _module_with(secretly_works=secretly_works) results = run(mod) - _, exc = results[0] + exc = results[0].outcome assert isinstance(exc, XPassed) assert exc.reason == 'supposedly broken' @@ -757,7 +756,7 @@ def both(): mod = _module_with(both=both) results = run(mod) - _, exc = results[0] + exc = results[0].outcome assert isinstance(exc, Skipped) assert exc.reason == 'skip me' assert called == [] @@ -768,12 +767,12 @@ def skip_on_parametrized_skips_every_combo(self): ) importlib.reload(mod) results = run(mod) - assert [name for name, _ in results] == [ + assert [r.name for r in results] == [ 'parametrized[0]', 'parametrized[1]', ] - for _, exc in results: - assert isinstance(exc, Skipped) - assert exc.reason == 'blocked' + for r in results: + assert isinstance(r.outcome, Skipped) + assert r.outcome.reason == 'blocked' assert mod.CALLS == [] def xfail_on_parametrized_evaluated_independently(self): @@ -781,7 +780,7 @@ def xfail_on_parametrized_evaluated_independently(self): 'tests.fixtures.runner.xfail_on_params', ) results = run(mod) - outcomes = dict(results) + outcomes = {r.name: r.outcome for r in results} # x == 1 raises -> XFailed assert isinstance(outcomes['parametrized[0]'], XFailed) assert isinstance(outcomes['parametrized[0]'].actual, ValueError) @@ -794,7 +793,7 @@ def skip_on_class_method_skips_method_only(self): ) mod.CALLS.clear() results = run(mod) - outcomes = dict(results) + outcomes = {r.name: r.outcome for r in results} assert isinstance( outcomes['Cls.skipped_method'], Skipped, ) @@ -813,7 +812,7 @@ def _module(self): def keep_none_runs_everything(self): results = run(self._module()) - names = sorted(name for name, _ in results) + names = sorted(r.name for r in results) assert names == [ 'SlowSuite.alpha', 'SlowSuite.beta', @@ -827,7 +826,7 @@ def include_filters_function_and_propagates_to_class(self): from testsweet._tag_filter import make_tag_filter keep = make_tag_filter(frozenset({'slow'}), frozenset()) results = run(self._module(), keep=keep) - names = sorted(name for name, _ in results) + names = sorted(r.name for r in results) # Both SlowSuite methods inherit the class's @tag('slow'). # The lone tagged function matches; everything else falls out. assert names == [ @@ -840,7 +839,7 @@ def include_matches_method_only_tag(self): from testsweet._tag_filter import make_tag_filter keep = make_tag_filter(frozenset({'db'}), frozenset()) results = run(self._module(), keep=keep) - names = sorted(name for name, _ in results) + names = sorted(r.name for r in results) # Untagged.gamma carries @tag('db') on the method; # SlowSuite.beta carries @tag('db') on top of the class's # @tag('slow'). @@ -850,7 +849,7 @@ def exclude_drops_class_and_function(self): from testsweet._tag_filter import make_tag_filter keep = make_tag_filter(frozenset(), frozenset({'slow'})) results = run(self._module(), keep=keep) - names = sorted(name for name, _ in results) + names = sorted(r.name for r in results) # Class-level @tag('slow') vetoes both SlowSuite methods; # the lone function falls out too. assert names == [ @@ -865,7 +864,7 @@ def exclude_overrides_include_when_both_apply(self): frozenset({'slow'}), frozenset({'db'}), ) results = run(self._module(), keep=keep) - names = sorted(name for name, _ in results) + names = sorted(r.name for r in results) # SlowSuite.alpha — slow only, kept. # SlowSuite.beta — slow+db, vetoed. # lone_function — slow only, kept. @@ -895,7 +894,86 @@ def filter_composes_with_names(self): ) # names= picks the Untagged class; keep= narrows to its # 'db'-tagged method. - assert [name for name, _ in results] == ['Untagged.gamma'] + assert [r.name for r in results] == ['Untagged.gamma'] + + +@test +class RunCapturesOutput: + def passing_test_stdout_captured_not_leaked(self): + @test + def talks(): + print('hello from test') + + mod = _module_with(talks=talks) + results = run(mod) + assert results[0].stdout == 'hello from test\n' + assert results[0].stderr == '' + assert isinstance(results[0].outcome, Passed) + + def stderr_captured_separately(self): + import sys + + @test + def warns(): + print('to err', file=sys.stderr) + + mod = _module_with(warns=warns) + results = run(mod) + assert results[0].stderr == 'to err\n' + assert results[0].stdout == '' + + def output_before_failure_is_captured(self): + @test + def noisy_fail(): + print('printed then failed') + assert False + + mod = _module_with(noisy_fail=noisy_fail) + results = run(mod) + assert isinstance(results[0].outcome, Failed) + assert results[0].stdout == 'printed then failed\n' + + def silent_test_has_empty_capture(self): + @test + def quiet(): + pass + + mod = _module_with(quiet=quiet) + results = run(mod) + assert results[0].stdout == '' + assert results[0].stderr == '' + + def skipped_test_has_empty_capture(self): + @test + @skip(reason='nope') + def skipped(): + print('never runs') # pragma: no cover + + mod = _module_with(skipped=skipped) + results = run(mod) + assert isinstance(results[0].outcome, Skipped) + assert results[0].stdout == '' + + def capture_is_inside_wrap_unit(self): + # Output printed inside wrap_unit's enter/exit (plugin + # setup/teardown) is NOT captured — only the unit's own output. + import sys + + @contextmanager + def wrap(name): + print('plugin enter', file=sys.stderr) + try: + yield + finally: + print('plugin exit', file=sys.stderr) + + @test + def unit(): + print('unit body', file=sys.stderr) + + mod = _module_with(unit=unit) + results = run(mod, wrap_unit=wrap) + assert results[0].stderr == 'unit body\n' def _module_with(**funcs): From c2ef2954a36b7e1c0333c3227892657d71a75930 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sun, 28 Jun 2026 01:25:56 +0100 Subject: [PATCH 07/10] Replay captured output in failure detail block --- src/testsweet/_report.py | 14 ++++++++++++ tests/report.py | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/testsweet/_report.py b/src/testsweet/_report.py index b9fc568..c14e73c 100644 --- a/src/testsweet/_report.py +++ b/src/testsweet/_report.py @@ -71,9 +71,20 @@ def format_result_line( return f'{full_name} ... {status}' +def _print_captured(stdout: str, stderr: str, file: TextIO) -> None: + if stdout: + print('-' * 26 + ' Captured stdout ' + '-' * 27, file=file) + print(stdout, end='' if stdout.endswith('\n') else '\n', file=file) + if stderr: + print('-' * 26 + ' Captured stderr ' + '-' * 27, file=file) + print(stderr, end='' if stderr.endswith('\n') else '\n', file=file) + + def print_failure_detail( full_name: str, outcome: Outcome, + stdout: str = '', + stderr: str = '', file: TextIO = sys.stdout, ) -> None: """Multi-line failure block. @@ -90,14 +101,17 @@ def print_failure_detail( print(f'XPASSED: {full_name}', file=file) print('-' * 70, file=file) print(_XPASS_DETAIL, file=file) + _print_captured(stdout, stderr, file) return case Failed(exc=exc): _print_traceback_block('FAIL', full_name, exc, file) explanation = explain_assertion(exc) if explanation is not None: print(explanation, file=file) + _print_captured(stdout, stderr, file) case Errored(exc=exc): _print_traceback_block('ERROR', full_name, exc, file) + _print_captured(stdout, stderr, file) def _print_traceback_block( diff --git a/tests/report.py b/tests/report.py index fb63a05..df1d13b 100644 --- a/tests/report.py +++ b/tests/report.py @@ -26,6 +26,14 @@ def _capture(full_name, outcome): return buf.getvalue() +def _capture_io(full_name, outcome, stdout='', stderr=''): + buf = io.StringIO() + print_failure_detail( + full_name, outcome, stdout=stdout, stderr=stderr, file=buf, + ) + return buf.getvalue() + + @test class FormatResultLine: def pass_outcome(self): @@ -109,6 +117,46 @@ def errored_emits_error_block(self): assert 'TypeError' in out +@test +class PrintFailureDetailCaptured: + def failed_shows_captured_stdout(self): + try: + assert 1 == 2 + except AssertionError as exc: + out = _capture_io('mod.t', Failed(exc), stdout='hello\n') + assert 'Captured stdout' in out + assert 'hello' in out + + def failed_shows_captured_stderr(self): + try: + assert 1 == 2 + except AssertionError as exc: + out = _capture_io('mod.t', Failed(exc), stderr='boom\n') + assert 'Captured stderr' in out + assert 'boom' in out + + def errored_shows_captured_output(self): + try: + raise TypeError('nope') + except TypeError as exc: + out = _capture_io('mod.t', Errored(exc), stdout='trace me\n') + assert 'Captured stdout' in out + assert 'trace me' in out + + def empty_capture_emits_no_section(self): + try: + assert 1 == 2 + except AssertionError as exc: + out = _capture_io('mod.t', Failed(exc)) + assert 'Captured stdout' not in out + assert 'Captured stderr' not in out + + def xpassed_shows_captured_output(self): + out = _capture_io('mod.t', XPassed('see #42'), stdout='ran anyway\n') + assert 'Captured stdout' in out + assert 'ran anyway' in out + + @test class FormatResultLineColor: def pass_has_green_ok(self): From 2393e7403b7f5141be817c715e9f60c2a6e253b7 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sun, 28 Jun 2026 01:42:01 +0100 Subject: [PATCH 08/10] Thread captured output through __main__ to failure reporting --- src/testsweet/__main__.py | 15 ++++++++++----- src/testsweet/_report.py | 4 +++- tests/fixtures/main/__init__.py | 0 tests/fixtures/main/capture_demo.py | 13 +++++++++++++ tests/main.py | 25 ++++++++++++++++++++++++- 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/main/__init__.py create mode 100644 tests/fixtures/main/capture_demo.py diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index 32b893c..26dbf6d 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -7,7 +7,7 @@ from testsweet._config import load_config from testsweet._loaders import scoped_sys_path from testsweet._plugins import load_plugins, session_for, unit_wrapper -from testsweet._outcomes import Errored, Failed, Outcome, Result, XPassed +from testsweet._outcomes import Errored, Failed, Result, XPassed from testsweet._report import ( format_result_line, print_failure_detail, @@ -123,7 +123,7 @@ def main(argv: list[str]) -> int: wrap_unit = unit_wrapper(plugins) use_color = _supports_color() results: list[Result] = [] - real_failures: list[tuple[str, Outcome]] = [] + real_failures: list[tuple[str, Result]] = [] start = time.monotonic() with session_for(plugins): groups = discover_targets(args.targets, config) @@ -150,10 +150,15 @@ def main(argv: list[str]) -> int: print(f'{indent}{format_result_line(short_name, result.outcome, use_color=use_color)}') results.append(result) if isinstance(result.outcome, (Failed, Errored, XPassed)): - real_failures.append((full_name, result.outcome)) + real_failures.append((full_name, result)) elapsed = time.monotonic() - start - for full_name, outcome in real_failures: - print_failure_detail(full_name, outcome) + for full_name, result in real_failures: + print_failure_detail( + full_name, + result.outcome, + result.stdout, + result.stderr, + ) print() print(summarize(results, use_color=use_color, elapsed=elapsed)) return 1 if real_failures else 0 diff --git a/src/testsweet/_report.py b/src/testsweet/_report.py index c14e73c..03ea900 100644 --- a/src/testsweet/_report.py +++ b/src/testsweet/_report.py @@ -85,13 +85,15 @@ def print_failure_detail( outcome: Outcome, stdout: str = '', stderr: str = '', - file: TextIO = sys.stdout, + file: TextIO | None = None, ) -> None: """Multi-line failure block. Fires only for ``Failed``, ``Errored``, and ``XPassed``. Other outcomes do not get a detail block. """ + if file is None: + file = sys.stdout match outcome: case Passed() | Skipped() | XFailed(): return diff --git a/tests/fixtures/main/__init__.py b/tests/fixtures/main/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/main/capture_demo.py b/tests/fixtures/main/capture_demo.py new file mode 100644 index 0000000..25d8322 --- /dev/null +++ b/tests/fixtures/main/capture_demo.py @@ -0,0 +1,13 @@ +from testsweet import test + + +@test +def passing_and_loud(): + print('SECRET_PASS_OUTPUT') + assert True + + +@test +def failing_and_loud(): + print('SECRET_FAIL_OUTPUT') + assert False diff --git a/tests/main.py b/tests/main.py index d4438eb..b477fac 100644 --- a/tests/main.py +++ b/tests/main.py @@ -1,8 +1,10 @@ +import io import os +from contextlib import redirect_stdout from unittest.mock import patch from testsweet import test -from testsweet.__main__ import _supports_color +from testsweet.__main__ import _supports_color, main def _tty_env(**env_overrides): @@ -61,3 +63,24 @@ def windows_ansicon_returns_true(self): _tty_env(ANSICON='80x24'): m.isatty.return_value = True assert _supports_color() + + +@test +class MainCapturesOutput: + def failing_test_output_is_replayed(self): + buf = io.StringIO() + with redirect_stdout(buf): + rc = main(['tests.fixtures.main.capture_demo']) + text = buf.getvalue() + assert rc == 1 + # The failing test's output is replayed under a capture section. + assert 'Captured stdout' in text + assert 'SECRET_FAIL_OUTPUT' in text + + def passing_test_output_is_suppressed(self): + buf = io.StringIO() + with redirect_stdout(buf): + main(['tests.fixtures.main.capture_demo']) + text = buf.getvalue() + # The passing test printed, but its output must not leak. + assert 'SECRET_PASS_OUTPUT' not in text From 79312a7f27533ac5d4fa8a46fa1b1fab6acb9687 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sun, 28 Jun 2026 01:59:27 +0100 Subject: [PATCH 09/10] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f80bfd..6cc6133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ Unreleased ### Improvements - Added tests and documentation for the assertion explainer. +- Added color output for outcomes. +- Grouped output by module and class. +- Included timing in the summary. +- Captured each test's stdout and stderr, replaying them in the + failure detail block. [0.2.2] (2026-06-27) From 9953f5d096d659076f7cd1fd2feb857a3cfbe0bc Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Sun, 28 Jun 2026 02:09:14 +0100 Subject: [PATCH 10/10] Lint --- src/testsweet/__main__.py | 7 ++++++- src/testsweet/_report.py | 6 +++++- tests/cli.py | 5 ++++- tests/main.py | 6 +++++- tests/report.py | 18 +++++++++++++++--- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/testsweet/__main__.py b/src/testsweet/__main__.py index 26dbf6d..8d38578 100644 --- a/src/testsweet/__main__.py +++ b/src/testsweet/__main__.py @@ -147,7 +147,12 @@ def main(argv: list[str]) -> int: print(f' {class_name}') last_class = class_name indent = ' ' if class_name else ' ' - print(f'{indent}{format_result_line(short_name, result.outcome, use_color=use_color)}') + result_line = format_result_line( + short_name, + result.outcome, + use_color=use_color, + ) + print(f'{indent}{result_line}') results.append(result) if isinstance(result.outcome, (Failed, Errored, XPassed)): real_failures.append((full_name, result)) diff --git a/src/testsweet/_report.py b/src/testsweet/_report.py index 03ea900..4a5d7b0 100644 --- a/src/testsweet/_report.py +++ b/src/testsweet/_report.py @@ -62,7 +62,11 @@ def format_result_line( return f'{full_name} ... {status}' case Failed(exc=exc): detail = str(exc) or assertion_source(exc) or '' - status = _c(f'FAIL: AssertionError: {detail}', _BOLD_RED, use_color) + status = _c( + f'FAIL: AssertionError: {detail}', + _BOLD_RED, + use_color, + ) return f'{full_name} ... {status}' case Errored(exc=exc): status = _c( diff --git a/tests/cli.py b/tests/cli.py index 6dfbeaa..cd6a525 100644 --- a/tests/cli.py +++ b/tests/cli.py @@ -532,7 +532,10 @@ def module_header_printed_once_per_module(self): result = _run_cli('tests.fixtures.runner.all_pass') assert result.returncode == 0 lines = result.stdout.splitlines() - header_lines = [l for l in lines if l == 'tests.fixtures.runner.all_pass'] + header_lines = [ + line for line in lines + if line == 'tests.fixtures.runner.all_pass' + ] assert len(header_lines) == 1 def result_lines_are_indented(self): diff --git a/tests/main.py b/tests/main.py index b477fac..f3e78d4 100644 --- a/tests/main.py +++ b/tests/main.py @@ -28,7 +28,11 @@ def no_color_env_returns_false(self): assert not _supports_color() def non_windows_tty_returns_true(self): - with patch('sys.stdout') as m, patch('sys.platform', 'linux'), _tty_env(): + with ( + patch('sys.stdout') as m, + patch('sys.platform', 'linux',), + _tty_env() + ): m.isatty.return_value = True assert _supports_color() diff --git a/tests/report.py b/tests/report.py index df1d13b..da92cec 100644 --- a/tests/report.py +++ b/tests/report.py @@ -165,12 +165,20 @@ def pass_has_green_ok(self): assert 'ok' in line def fail_has_red_status(self): - line = format_result_line('mod.t', Failed(AssertionError('x')), use_color=True) + line = format_result_line( + 'mod.t', + Failed(AssertionError('x')), + use_color=True, + ) assert '\x1b[' in line assert 'FAIL' in line def error_has_red_status(self): - line = format_result_line('mod.t', Errored(TypeError('x')), use_color=True) + line = format_result_line( + 'mod.t', + Errored(TypeError('x')), + use_color=True, + ) assert '\x1b[' in line assert 'ERROR' in line @@ -180,7 +188,11 @@ def skipped_has_yellow_status(self): assert 'skipped' in line def xfailed_has_yellow_status(self): - line = format_result_line('mod.t', XFailed(ValueError('x')), use_color=True) + line = format_result_line( + 'mod.t', + XFailed(ValueError('x')), + use_color=True, + ) assert '\x1b[33m' in line assert 'xfailed' in line