From b01c1957367b656d59b0ae2fb96a402ef75220c5 Mon Sep 17 00:00:00 2001 From: "v.razuvaev" Date: Sun, 20 Sep 2026 04:08:40 +0300 Subject: [PATCH 1/2] Core 0.12: exhaustive(), exhaustiveBudget(), flakyReplays() and searchRuns() on the chain; PROPERTY_EXHAUSTIVE / PROPERTY_SEARCH_RUNS; table, exhaustive and search reports; TargetImproved in the verbose trace --- CHANGELOG.md | 17 ++++ README.md | 23 +++++- README.ru.md | 22 ++++- composer.json | 2 +- llms.txt | 6 ++ src/PhpUnit/PropertyCheck.php | 141 ++++++++++++++++++++++++++++++++ src/PhpUnit/VerboseListener.php | 11 +++ tests/EnvironmentParityTest.php | 61 ++++++++++++++ tests/PropertyCheckTest.php | 117 ++++++++++++++++++++++++++ tests/Support/Env.php | 2 + 10 files changed, 395 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ddb62..6a7b0b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## Unreleased + +- **Added:** the core 0.12 knobs on the chain — `exhaustive()` / + `exhaustiveBudget()` (walk the parameter domain instead of sampling it), + `flakyReplays()` (re-execute the minimised counterexample; a passing replay + is reported as flaky) and `searchRuns()` (targeted search after the random + phase for a body that calls `Target::maximize()`/`minimize()`); each + validates at the call, naming the property, like the other setters. +- **Added:** `PROPERTY_EXHAUSTIVE` (a flag) and `PROPERTY_SEARCH_RUNS` (a + non-negative integer, `0` switches the search off) dial the suite, the way + `PROPERTY_DERANDOMIZE` and `PROPERTY_RUNS` do. +- **Added:** every `Classify::tabulate()` table with its pairwise + intersections, the exhaustive outcome (walked, on stdout; declined and why, + on stderr) and the search report are printed beside the distribution line; + `PROPERTY_VERBOSE` logs every `TargetImproved` event. +- Requires `rasuvaeff/property-testing-core` `^0.12`. + ## 0.9.2 — 2026-09-19 - **Fixed.** A wide `throws()` swallowed a failed assertion: PHPUnit's diff --git a/README.md b/README.md index 40b40c7..8cc0fbe 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ read — inside an ordinary PHPUnit `TestCase`. - PHP 8.3+ - [`phpunit/phpunit`](https://packagist.org/packages/phpunit/phpunit) `^11.5 || ^12.0 || ^13.0` -- [`rasuvaeff/property-testing-core`](https://packagist.org/packages/rasuvaeff/property-testing-core) `^0.9 || ^0.10 || ^0.11` +- [`rasuvaeff/property-testing-core`](https://packagist.org/packages/rasuvaeff/property-testing-core) `^0.12` PHPUnit 13 requires PHP 8.4.1 or newer. On PHP 8.3, Composer resolves a compatible PHPUnit 11 or 12 release. @@ -113,12 +113,17 @@ Reproduce the exact run by pinning the reported seed: `->seed(7382910)`. | `edgeCases(EdgeCases)` | `None` turns off the numeric boundary bias — for a property the edges only cost runs | | `auto(bool = true)` | Derives generators from the closure's signature for every parameter the `forAll()` map does not cover; the map becomes partial overrides. Off by default, and stays off | | `throws(string)` | The exception class every trial must throw — a trial that throws it passes, one that does not (or throws another class) fails and shrinks. The property-level replacement for `expectException()`, which never sees a throw from inside the body | +| `exhaustive(bool = true)` | Walk the whole parameter domain instead of sampling it when every generator is `Enumerable` and the product fits the budget; otherwise the phase samples and a warning says why. `runs()` is ignored when it walks — see the [core README](https://github.com/rasuvaeff/property-testing-core#exhaustive-mode) | +| `exhaustiveBudget(int)` | The largest domain `exhaustive()` walks (default 10 000) | +| `flakyReplays(int)` | Re-executions of the minimised counterexample (default 2); one that passes marks it flaky, with a `Flaky:` line in the failure. `0` disables — see [flaky detection](https://github.com/rasuvaeff/property-testing-core#flaky-detection) | +| `searchRuns(int)` | Bodies the targeted search may execute after the random phase, for a body that calls `Target::maximize()`/`minimize()` (default 0 — no search) — see [targeted search](https://github.com/rasuvaeff/property-testing-core#targeted-search-target) | Every setter validates its argument at the call, with the property's name in the message: `runs(0)` throws `Property "testSortIsIdempotent": runs must be greater than or equal to 1` -(the same for `maxShrinks`/`maxDiscards` below `0` and for -`timeoutMs`/`budgetMs`/`shrinkBudgetMs` below `1`). A `path()` without a +(the same for `maxShrinks`/`maxDiscards`/`flakyReplays`/`searchRuns` below +`0` and for `timeoutMs`/`budgetMs`/`shrinkBudgetMs`/`exhaustiveBudget` below +`1`). A `path()` without a `seed()` — in either order — or a `PROPERTY_PATH` without a seed is refused by `check()` the same way. The `forAll()` map is checked by `check()` too, before the engine runs: a value that is not an `ArbitraryInterface` throws @@ -291,6 +296,8 @@ Byte-for-byte parity with the Testo adapter — one contract across adapters: | `PROPERTY_DERANDOMIZE` | Derives every unset seed from the property id, making a whole suite reproducible without editing it. Same switch words as `PROPERTY_VERBOSE`: `''` is unset, `0`/`false`/`off`/`no` are off, anything else is on | | `PROPERTY_PATH` | A recorded shrink descent (`CounterExample::$path`) replayed instead of searched for. Needs the seed that produced it; an explicit `path()` wins. It describes one failure, so run it with `--filter` on that one test — every other property would report the path as stale | | `PROPERTY_EDGE_CASES` | `mixin` or `none` (case-insensitive) — the numeric boundary bias for the whole suite, overriding `edgeCases()`. An unknown value throws | +| `PROPERTY_EXHAUSTIVE` | Turns exhaustive mode on for every property whose domain fits its budget, overriding `exhaustive()`; the same switch words as `PROPERTY_DERANDOMIZE` | +| `PROPERTY_SEARCH_RUNS` | Non-negative integer overriding `searchRuns()` for the whole suite — a bigger search budget on a nightly, or `0` to switch it off. A malformed value throws | `PROPERTY_DB` takes either a directory or a Redis DSN: @@ -335,6 +342,16 @@ Property "testSortKeepsEveryElement" distribution: long 39% (77/200), short 61% A property that discards more than 90% of its attempts (via `Assume::that()`) gets a warning suggesting narrower generators. +Beside the distribution line, the adapter prints what else the engine +measured: each `Classify::tabulate()` table with its tag shares and the pairs +hit together (`Property "…" table features: compressed 40% (80/200), retried +15% (30/200); together: compressed & retried 10% (20/200)`), whether +exhaustive mode walked the domain (`enumerated its whole domain of 24 +input(s)`, stdout) or why it sampled instead (stderr), and the search report +of a body that targets something (`search: 100 evaluation(s); delay max 58210 +(7 improvement(s))`). `PROPERTY_VERBOSE` also logs every `TargetImproved` +event with the input that scored it. + These diagnostics — the distribution on stdout, the discard warning and the unstable-id warnings on stderr — are written straight to the process streams while PHPUnit is printing its progress dots, so each one starts with a newline diff --git a/README.ru.md b/README.ru.md index d12609e..4a6047c 100644 --- a/README.ru.md +++ b/README.ru.md @@ -31,7 +31,7 @@ shrink до минимального контрпримера, который р - PHP 8.3+ - [`phpunit/phpunit`](https://packagist.org/packages/phpunit/phpunit) `^11.5 || ^12.0 || ^13.0` -- [`rasuvaeff/property-testing-core`](https://packagist.org/packages/rasuvaeff/property-testing-core) `^0.9 || ^0.10 || ^0.11` +- [`rasuvaeff/property-testing-core`](https://packagist.org/packages/rasuvaeff/property-testing-core) `^0.12` PHPUnit 13 требует PHP 8.4.1 или новее. На PHP 8.3 Composer выбирает совместимый релиз PHPUnit 11 или 12. @@ -113,12 +113,16 @@ Property falsified after 12 successful run(s); seed=7382910 | `edgeCases(EdgeCases)` | `None` выключает граничное смещение числовых генераторов — для property, которой края стоят только прогонов | | `auto(bool = true)` | Достраивает генераторы из сигнатуры кложуры для параметров, не покрытых картой `forAll()`; карта становится частичными overrides. По умолчанию выключен и дефолтом не станет | | `throws(string)` | Класс исключения, которое обязан бросить каждый trial: бросивший его trial проходит, не бросивший (или бросивший другой класс) валит property и shrink-ается. Замена `expectException()` на уровне property — сам `expectException()` бросок из тела не видит | +| `exhaustive(bool = true)` | Обойти весь домен параметров вместо выборки, когда каждый генератор — `Enumerable`, а произведение умещается в бюджет; иначе фаза делает выборку, а предупреждение говорит почему. `runs()` при обходе игнорируется — см. [README core](https://github.com/rasuvaeff/property-testing-core#exhaustive-mode) | +| `exhaustiveBudget(int)` | Наибольший домен, который обходит `exhaustive()` (по умолчанию 10 000) | +| `flakyReplays(int)` | Повторные исполнения минимизированного контрпримера (по умолчанию 2); прошедший помечает его flaky, со строкой `Flaky:` в сообщении. `0` выключает — см. [детекцию flaky](https://github.com/rasuvaeff/property-testing-core#flaky-detection) | +| `searchRuns(int)` | Сколько тел может исполнить целевой поиск после random-фазы, если тело вызывает `Target::maximize()`/`minimize()` (по умолчанию 0 — без поиска) — см. [целевой поиск](https://github.com/rasuvaeff/property-testing-core#targeted-search-target) | Каждый сеттер проверяет аргумент в момент вызова, с именем property в сообщении: `runs(0)` бросает `Property "testSortIsIdempotent": runs must be greater than or equal to 1` -(так же `maxShrinks`/`maxDiscards` ниже `0` и `timeoutMs`/`budgetMs`/ -`shrinkBudgetMs` ниже `1`). `path()` без `seed()` — в любом порядке — или +(так же `maxShrinks`/`maxDiscards`/`flakyReplays`/`searchRuns` ниже `0` и +`timeoutMs`/`budgetMs`/`shrinkBudgetMs`/`exhaustiveBudget` ниже `1`). `path()` без `seed()` — в любом порядке — или `PROPERTY_PATH` без seed так же отвергает `check()`. Карту `forAll()` `check()` тоже проверяет до запуска движка: значение, не являющееся `ArbitraryInterface`, даёт @@ -289,6 +293,8 @@ $this->forAll(['width' => Gen::intBetween(1, 5000), 'minWidth' => Gen::intBetwee | `PROPERTY_DERANDOMIZE` | Выводит каждый незаданный seed из id property: весь сьют становится воспроизводимым без правки кода. Те же слова-переключатели, что у `PROPERTY_VERBOSE`: `''` — не задано, `0`/`false`/`off`/`no` — выключено, всё остальное — включено | | `PROPERTY_PATH` | Записанный спуск shrink (`CounterExample::$path`) воспроизводится вместо повторного поиска. Нужен seed того прогона; явный `path()` побеждает. Он описывает одно падение, поэтому запускайте с `--filter` на этот один тест — любое другое property сообщит, что путь устарел | | `PROPERTY_EDGE_CASES` | `mixin` или `none` (регистр не важен) — граничное смещение для всего сьюта, перекрывает `edgeCases()`. Неизвестное значение — исключение | +| `PROPERTY_EXHAUSTIVE` | Включает исчерпывающий режим для каждой property, чей домен умещается в бюджет, перекрывая `exhaustive()`; те же слова-переключатели, что у `PROPERTY_DERANDOMIZE` | +| `PROPERTY_SEARCH_RUNS` | Неотрицательное целое, перекрывающее `searchRuns()` для всего сьюта — больший бюджет поиска на ночном прогоне или `0`, чтобы выключить его. Испорченное значение — исключение | `PROPERTY_DB` принимает либо каталог, либо Redis-DSN: @@ -332,6 +338,16 @@ Property "testSortKeepsEveryElement" distribution: long 39% (77/200), short 61% Property, отбрасывающая больше 90% попыток (через `Assume::that()`), получает предупреждение с советом сузить генераторы. +Рядом со строкой распределения адаптер печатает остальное, что измерил +движок: каждую таблицу `Classify::tabulate()` с долями тегов и парами, +встретившимися вместе (`Property "…" table features: compressed 40% (80/200), +retried 15% (30/200); together: compressed & retried 10% (20/200)`), обошёл ли +исчерпывающий режим домен (`enumerated its whole domain of 24 input(s)`, +stdout) или почему сделал выборку (stderr), и отчёт поиска property, которая +что-то целит (`search: 100 evaluation(s); delay max 58210 (7 improvement(s))`). +`PROPERTY_VERBOSE` дополнительно логирует каждое событие `TargetImproved` с +давшим его входом. + Эта диагностика — распределение в stdout, предупреждение о discard'ах и предупреждения о нестабильном id в stderr — пишется прямо в потоки процесса, пока PHPUnit печатает точки прогресса, поэтому каждая строка начинается с diff --git a/composer.json b/composer.json index ef504df..9639d10 100644 --- a/composer.json +++ b/composer.json @@ -25,7 +25,7 @@ "require": { "php": "8.3 - 8.5", "phpunit/phpunit": "^11.5 || ^12.0 || ^13.0", - "rasuvaeff/property-testing-core": "^0.9 || ^0.10 || ^0.11" + "rasuvaeff/property-testing-core": "^0.12" }, "require-dev": { "ergebnis/composer-normalize": "^2.51", diff --git a/llms.txt b/llms.txt index ffe9464..f7b6e0a 100644 --- a/llms.txt +++ b/llms.txt @@ -83,6 +83,10 @@ public function phases(array $phases): self; // list: Examples, C public function derandomize(bool $on = true): self; // unset seed derived from the property id public function path(string $path): self; // replay a recorded shrink descent; needs its seed public function edgeCases(EdgeCases $mode): self; // Mixin (default) | None = no numeric boundary bias +public function exhaustive(bool $exhaustive = true): self; // walk the whole parameter product when every generator is Enumerable and it fits the budget +public function exhaustiveBudget(int $budget): self; // >= 1; default 10_000 +public function flakyReplays(int $replays): self; // >= 0; default 2; a passing replay of the minimised input = flaky (Flaky: line) +public function searchRuns(int $runs): self; // >= 0; default 0 = no targeted search after the random phase (Target::maximize()/minimize()) public function seed(int $seed): self; // pin the random phase; disables corpus replay public function maxShrinks(int $maxShrinks): self; // cap accepted shrink steps; 0 = no shrinking public function maxDiscards(int $maxDiscards): self; // discard budget; default runs * 10 @@ -143,6 +147,8 @@ final class SortPropertyTest extends TestCase | `PROPERTY_DERANDOMIZE` | same switch words as `PROPERTY_VERBOSE` | Unset seeds derived from the property id; `seed()`/`PROPERTY_SEED` still win | | `PROPERTY_PATH` | recorded `CounterExample::$path` | Replays that shrink descent; needs the seed of that run; explicit `path()` wins | | `PROPERTY_EDGE_CASES` | `mixin` \| `none` (case-insensitive) | Numeric boundary bias; OVERRIDES `edgeCases()`; unknown value throws | +| `PROPERTY_EXHAUSTIVE` | flag (same words as `PROPERTY_DERANDOMIZE`) | Exhaustive mode for the suite; OVERRIDES `exhaustive()` | +| `PROPERTY_SEARCH_RUNS` | non-negative integer (`0` = off) | Search budget for the suite; OVERRIDES `searchRuns()`; malformed throws | ### Classification output diff --git a/src/PhpUnit/PropertyCheck.php b/src/PhpUnit/PropertyCheck.php index 671a5a0..89a03ef 100644 --- a/src/PhpUnit/PropertyCheck.php +++ b/src/PhpUnit/PropertyCheck.php @@ -14,17 +14,22 @@ use Rasuvaeff\PropertyTesting\Runner\Corpus; use Rasuvaeff\PropertyTesting\Runner\CorpusFactory; use Rasuvaeff\PropertyTesting\Runner\CoverageFailed; +use Rasuvaeff\PropertyTesting\Runner\DistributionReport; use Rasuvaeff\PropertyTesting\Runner\EdgeCases; use Rasuvaeff\PropertyTesting\Runner\EnvironmentOverrides; use Rasuvaeff\PropertyTesting\Runner\GaveUp; +use Rasuvaeff\PropertyTesting\Runner\LabelShare; use Rasuvaeff\PropertyTesting\Runner\Passed; use Rasuvaeff\PropertyTesting\Runner\Phase; use Rasuvaeff\PropertyTesting\Runner\PropertyConfig; use Rasuvaeff\PropertyTesting\Runner\PropertyDefinition; use Rasuvaeff\PropertyTesting\Runner\PropertyRunner; use Rasuvaeff\PropertyTesting\Runner\RunStatistics; +use Rasuvaeff\PropertyTesting\Runner\SearchReport; use Rasuvaeff\PropertyTesting\Runner\ShrinkMode; +use Rasuvaeff\PropertyTesting\Runner\TargetOutcome; use Rasuvaeff\PropertyTesting\Runner\TimeBudgetExceeded; +use Rasuvaeff\PropertyTesting\ValueRenderer; /** * Fluent builder mapping the engine's structured PropertyResult onto PHPUnit: @@ -60,6 +65,14 @@ final class PropertyCheck private ?int $shrinkBudgetMs = null; private ?string $path = null; private ?EdgeCases $edgeCases = null; + + private ?bool $exhaustive = null; + + private int $exhaustiveBudget = 10_000; + + private int $flakyReplays = 2; + + private ?int $searchRuns = null; private ?bool $derandomize = null; private bool $auto = false; private ?string $expectedExceptionClass = null; @@ -315,6 +328,58 @@ public function edgeCases(EdgeCases $edgeCases): self return $this; } + /** + * Walk the whole parameter domain instead of sampling it, when every + * generator has a finite domain (`Enumerable`) and the product fits + * {@see exhaustiveBudget()}; otherwise the phase samples and a warning + * says why. {@see runs()} is ignored when it walks. `PROPERTY_EXHAUSTIVE` + * turns it on for the whole suite. + */ + public function exhaustive(bool $exhaustive = true): self + { + $this->exhaustive = $exhaustive; + + return $this; + } + + /** + * The largest domain {@see exhaustive()} walks; 10 000 by default. + */ + public function exhaustiveBudget(int $exhaustiveBudget): self + { + $this->assertAtLeast('exhaustiveBudget', $exhaustiveBudget, 1); + $this->exhaustiveBudget = $exhaustiveBudget; + + return $this; + } + + /** + * Re-executions of the minimised counterexample after the descent — one + * that passes marks the counterexample flaky, with a `Flaky:` line in the + * failure. 2 by default; 0 disables the check. + */ + public function flakyReplays(int $flakyReplays): self + { + $this->assertAtLeast('flakyReplays', $flakyReplays, 0); + $this->flakyReplays = $flakyReplays; + + return $this; + } + + /** + * Bodies the targeted search may execute after the random phase, for a + * body that calls `Target::maximize()`/`minimize()`: the best-scoring + * inputs are mutated one parameter at a time. 0 (the default) performs no + * search. `PROPERTY_SEARCH_RUNS` overrides it for the whole suite. + */ + public function searchRuns(int $searchRuns): self + { + $this->assertAtLeast('searchRuns', $searchRuns, 0); + $this->searchRuns = $searchRuns; + + return $this; + } + /** * Replays the shrink descent of an earlier failure, as reported by * `CounterExample::$path`, instead of searching for it again. It needs the @@ -453,6 +518,12 @@ public function check(\Closure $property): void derandomize: EnvironmentOverrides::flag(getenv('PROPERTY_DERANDOMIZE')) ?? $this->derandomize ?? false, path: $path, edgeCases: EnvironmentOverrides::edgeCases(getenv('PROPERTY_EDGE_CASES')) ?? $this->edgeCases ?? EdgeCases::Mixin, + // Suite dials as well: enumerate everything small on a + // nightly, or give the search a bigger budget there. + exhaustive: EnvironmentOverrides::flag(getenv('PROPERTY_EXHAUSTIVE')) ?? $this->exhaustive ?? false, + exhaustiveBudget: $this->exhaustiveBudget, + flakyReplays: $this->flakyReplays, + searchRuns: EnvironmentOverrides::count('PROPERTY_SEARCH_RUNS', getenv('PROPERTY_SEARCH_RUNS')) ?? $this->searchRuns ?? 0, ), examples: $this->examples, replayRegressions: $this->seed === null, @@ -499,6 +570,9 @@ public function check(\Closure $property): void // suite merges the two streams; see AGENTS.md. $this->warnOnExcessiveSkips($statistics); $this->reportClassifications($statistics); + $this->reportTables($statistics); + $this->reportExhaustive($statistics); + $this->reportSearch($statistics); } // The check is the assertion, whichever way it went: counted before @@ -614,6 +688,73 @@ private function reportClassifications(RunStatistics $statistics): void $this->diagnose($this->stdout, sprintf('Property "%s" distribution: %s', $this->name, implode(', ', $parts))); } + /** + * One line per `Classify::tabulate()` table: the tag shares, then the + * pairs hit together when any were — the same wording the Testo adapter + * prints, on the distribution's stream. + */ + private function reportTables(RunStatistics $statistics): void + { + if ($statistics->checks <= 0) { + return; + } + + $report = DistributionReport::of($statistics, coverageAssessed: true); + + foreach ($report->tables as $table => $shares) { + $render = static fn(LabelShare $share): string => sprintf('%s %d%% (%d/%d)', $share->label, (int) round($share->percent), $share->count, $report->checks); + $parts = array_map($render, $shares); + $pairs = array_map($render, $report->intersections[$table] ?? []); + + $this->diagnose( + $this->stdout, + sprintf('Property "%s" table %s: %s', $this->name, $table, implode(', ', $parts)) + . ($pairs === [] ? '' : sprintf('; together: %s', implode(', ', $pairs))), + ); + } + } + + /** + * What exhaustive mode did, for a property that asked for it: the domain + * it walked (stdout, like the distribution), or why it sampled instead + * (stderr, like the discard warning). + */ + private function reportExhaustive(RunStatistics $statistics): void + { + if ($statistics->domainSize !== null) { + $this->diagnose($this->stdout, sprintf('Property "%s" enumerated its whole domain of %d input(s)', $this->name, $statistics->domainSize)); + } elseif ($statistics->exhaustiveDeclined !== null) { + $this->diagnose($this->stderr, sprintf('Property "%s" could not enumerate its domain and sampled instead: %s', $this->name, $statistics->exhaustiveDeclined)); + } + } + + /** + * The search report: how many bodies the search phase executed and where + * every `Target` label ended up. + */ + private function reportSearch(RunStatistics $statistics): void + { + $search = $statistics->search; + + if (!$search instanceof SearchReport) { + return; + } + + $parts = array_map( + static fn(TargetOutcome $target): string => sprintf( + '%s %s %s (%d improvement(s)%s)', + $target->label, + $target->direction->value === 'maximize' ? 'max' : 'min', + $target->best === null ? '-' : ValueRenderer::render($target->best), + $target->improvements, + $target->recalled > 0 ? sprintf(', %d recalled', $target->recalled) : '', + ), + $search->targets, + ); + + $this->diagnose($this->stdout, sprintf('Property "%s" search: %d evaluation(s); %s', $this->name, $search->evaluations, implode(', ', $parts))); + } + /** * An id derived from a closure keys the corpus by something that moves — * the engine names the problem, this prints it. Same channel as the diff --git a/src/PhpUnit/VerboseListener.php b/src/PhpUnit/VerboseListener.php index 9d54668..14693aa 100644 --- a/src/PhpUnit/VerboseListener.php +++ b/src/PhpUnit/VerboseListener.php @@ -10,6 +10,7 @@ use Rasuvaeff\PropertyTesting\Event\RunPassed; use Rasuvaeff\PropertyTesting\Event\RunStarted; use Rasuvaeff\PropertyTesting\Event\ShrinkAccepted; +use Rasuvaeff\PropertyTesting\Event\TargetImproved; use Rasuvaeff\PropertyTesting\PropertyListener; use Rasuvaeff\PropertyTesting\ValueRenderer; @@ -53,6 +54,16 @@ public function onEvent(PropertyEvent $event): void $this->formatArguments($event->draws), )); } + } elseif ($event instanceof TargetImproved) { + $this->log(sprintf( + 'Property "%s" target %s %s: %s -> %s (%s)', + $this->name($event->propertyId), + $event->label, + $event->direction->value === 'maximize' ? 'max' : 'min', + $event->previous === null ? '-' : ValueRenderer::render($event->previous), + ValueRenderer::render($event->score), + $this->formatArguments($event->arguments), + )); } elseif ($event instanceof ShrinkAccepted) { $this->log(sprintf( 'Property "%s" shrink step %d: %s=%s -> %s', diff --git a/tests/EnvironmentParityTest.php b/tests/EnvironmentParityTest.php index e9843c2..32cbece 100644 --- a/tests/EnvironmentParityTest.php +++ b/tests/EnvironmentParityTest.php @@ -23,6 +23,7 @@ use Rasuvaeff\PropertyTesting\Runner\EdgeCases; use Rasuvaeff\PropertyTesting\Runner\EnvironmentOverrides; use Rasuvaeff\PropertyTesting\Runner\Phase; +use Rasuvaeff\PropertyTesting\Target; /** * The environment contract, byte-for-byte the Testo adapter's: PROPERTY_RUNS @@ -719,6 +720,66 @@ public function testPropertyEdgeCasesCanTurnThemBackOn(): void self::assertGreaterThan(10, $this->edgesSeen(static fn(PropertyCheck $check): PropertyCheck => $check->edgeCases(EdgeCases::None))); } + public function testPropertyExhaustiveTurnsEnumerationOnForTheSuite(): void + { + putenv('PROPERTY_EXHAUSTIVE=1'); + $seen = 0; + + $this->forAll(['value' => Gen::intBetween(1, 10)]) + ->output(fopen('php://memory', 'r+'), fopen('php://memory', 'r+')) + ->runs(3) + ->check(static function (int $value) use (&$seen): void { + ++$seen; + }); + + self::assertSame(10, $seen); + } + + public function testPropertyExhaustiveCanTurnTheChainOff(): void + { + putenv('PROPERTY_EXHAUSTIVE=off'); + $seen = 0; + + $this->forAll(['value' => Gen::intBetween(1, 10)]) + ->runs(3) + ->seed(1) + ->exhaustive() + ->check(static function (int $value) use (&$seen): void { + ++$seen; + }); + + self::assertSame(3, $seen); + } + + public function testPropertySearchRunsOverridesTheChain(): void + { + putenv('PROPERTY_SEARCH_RUNS=7'); + $seen = 0; + + $this->forAll(['value' => Gen::intBetween(1, 10)]) + ->output(fopen('php://memory', 'r+'), fopen('php://memory', 'r+')) + ->runs(3) + ->seed(1) + ->searchRuns(50) + ->check(static function (int $value) use (&$seen): void { + ++$seen; + Target::maximize('value', $value); + }); + + self::assertSame(10, $seen); + } + + public function testPropertySearchRunsRejectsAMalformedValue(): void + { + putenv('PROPERTY_SEARCH_RUNS=many'); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('PROPERTY_SEARCH_RUNS must be a non-negative integer, got "many"'); + + $this->forAll(['value' => Gen::intBetween(0, 10)]) + ->check(static function (int $value): void {}); + } + public function testPropertyEdgeCasesRejectsAnUnknownValue(): void { // Spaces on purpose: the message quotes the trimmed value, so a diff --git a/tests/PropertyCheckTest.php b/tests/PropertyCheckTest.php index 36b27e0..3121f2c 100644 --- a/tests/PropertyCheckTest.php +++ b/tests/PropertyCheckTest.php @@ -29,6 +29,7 @@ use Rasuvaeff\PropertyTesting\PropertyViolationException; use Rasuvaeff\PropertyTesting\Runner\Phase; use Rasuvaeff\PropertyTesting\Runner\ShrinkMode; +use Rasuvaeff\PropertyTesting\Target; use Rasuvaeff\PropertyTesting\TimeBudgetExceededException; #[CoversClass(PropertyCheck::class)] @@ -922,6 +923,9 @@ public static function rejectedSetterValueProvider(): iterable yield 'timeoutMs(0)' => [static fn(PropertyCheck $check): PropertyCheck => $check->timeoutMs(0), 'timeoutMs must be greater than or equal to 1']; yield 'budgetMs(0)' => [static fn(PropertyCheck $check): PropertyCheck => $check->budgetMs(0), 'budgetMs must be greater than or equal to 1']; yield 'shrinkBudgetMs(0)' => [static fn(PropertyCheck $check): PropertyCheck => $check->shrinkBudgetMs(0), 'shrinkBudgetMs must be greater than or equal to 1']; + yield 'exhaustiveBudget(0)' => [static fn(PropertyCheck $check): PropertyCheck => $check->exhaustiveBudget(0), 'exhaustiveBudget must be greater than or equal to 1']; + yield 'flakyReplays(-1)' => [static fn(PropertyCheck $check): PropertyCheck => $check->flakyReplays(-1), 'flakyReplays must be greater than or equal to 0']; + yield 'searchRuns(-1)' => [static fn(PropertyCheck $check): PropertyCheck => $check->searchRuns(-1), 'searchRuns must be greater than or equal to 0']; } public function testEverySetterAcceptsItsMinimum(): void @@ -951,6 +955,119 @@ public function testEverySetterAcceptsItsMinimum(): void ->check(static function (int $value): void { self::assertGreaterThanOrEqual(0, $value); }); + + $this->forAll(['value' => Gen::intBetween(0, 10)]) + ->runs(1) + ->exhaustiveBudget(1) + ->flakyReplays(0) + ->searchRuns(0) + ->seed(1) + ->check(static function (int $value): void { + self::assertGreaterThanOrEqual(0, $value); + }); + } + + public function testExhaustiveWalksTheWholeDomainAndSaysSo(): void + { + $stdout = fopen('php://memory', 'r+'); + $stderr = fopen('php://memory', 'r+'); + \assert(\is_resource($stdout) && \is_resource($stderr)); + $seen = []; + + $this->forAll(['flag' => Gen::bool(), 'n' => Gen::intBetween(0, 2)]) + ->runs(2) + ->seed(1) + ->exhaustive() + ->output($stdout, $stderr) + ->check(static function (bool $flag, int $n) use (&$seen): void { + $seen[] = [$flag, $n]; + }); + + self::assertSame([[false, 0], [false, 1], [false, 2], [true, 0], [true, 1], [true, 2]], $seen); + rewind($stdout); + self::assertSame("\n" . 'Property "testExhaustiveWalksTheWholeDomainAndSaysSo" enumerated its whole domain of 6 input(s)' . "\n", (string) stream_get_contents($stdout)); + rewind($stderr); + self::assertSame('', (string) stream_get_contents($stderr)); + } + + public function testExhaustiveThatDeclinesWarnsWhy(): void + { + $stdout = fopen('php://memory', 'r+'); + $stderr = fopen('php://memory', 'r+'); + \assert(\is_resource($stdout) && \is_resource($stderr)); + + $this->forAll(['flag' => Gen::bool(), 'n' => Gen::intBetween(0, 2)]) + ->runs(2) + ->seed(1) + ->exhaustive() + ->exhaustiveBudget(5) + ->output($stdout, $stderr) + ->check(static function (bool $flag, int $n): void {}); + + rewind($stderr); + self::assertSame("\n" . 'Property "testExhaustiveThatDeclinesWarnsWhy" could not enumerate its domain and sampled instead: the domain has 6 inputs, above the exhaustive budget of 5' . "\n", (string) stream_get_contents($stderr)); + } + + public function testTabulatedCategoriesArePrintedWithTheirIntersections(): void + { + $stdout = fopen('php://memory', 'r+'); + $stderr = fopen('php://memory', 'r+'); + \assert(\is_resource($stdout) && \is_resource($stderr)); + + $this->forAll(['value' => Gen::intBetween(0, 10)]) + ->runs(4) + ->seed(3) + ->output($stdout, $stderr) + ->check(static function (int $value): void { + Classify::tabulate('features', ['a', 'b']); + }); + + rewind($stdout); + self::assertSame("\n" . 'Property "testTabulatedCategoriesArePrintedWithTheirIntersections" table features: a 100% (4/4), b 100% (4/4); together: a & b 100% (4/4)' . "\n", (string) stream_get_contents($stdout)); + } + + public function testTheSearchReportIsPrintedForATargetingBody(): void + { + $stdout = fopen('php://memory', 'r+'); + $stderr = fopen('php://memory', 'r+'); + \assert(\is_resource($stdout) && \is_resource($stderr)); + + $this->forAll(['a' => Gen::intBetween(0, 1000), 'b' => Gen::intBetween(0, 1000)]) + ->runs(10) + ->seed(3) + ->searchRuns(20) + ->output($stdout, $stderr) + ->check(static function (int $a, int $b): void { + Target::maximize('sum', $a + $b); + }); + + rewind($stdout); + self::assertSame(1, preg_match( + '/^\nProperty "\w+" search: 20 evaluation\(s\); sum max \d+ \(\d+ improvement\(s\)\)\n$/', + (string) stream_get_contents($stdout), + )); + } + + public function testAFlakyCounterexampleIsNamedInTheFailure(): void + { + $failed = false; + + try { + $this->forAll(['value' => Gen::intBetween(50, 100)]) + ->runs(5) + ->seed(2) + ->check(static function (int $value) use (&$failed): void { + if (!$failed) { + $failed = true; + + self::fail('once'); + } + }); + + self::fail('Expected the property to be falsified'); + } catch (AssertionFailedError $failure) { + self::assertStringContainsString('Flaky: the minimised input passed on replay 1;', $failure->getMessage()); + } } public function testASetterErrorLeavesTheEngineMessageOutOfIt(): void diff --git a/tests/Support/Env.php b/tests/Support/Env.php index 4fe89c2..2268448 100644 --- a/tests/Support/Env.php +++ b/tests/Support/Env.php @@ -73,6 +73,8 @@ public static function isolateProperty(): \Closure 'PROPERTY_DERANDOMIZE' => null, 'PROPERTY_PATH' => null, 'PROPERTY_EDGE_CASES' => null, + 'PROPERTY_EXHAUSTIVE' => null, + 'PROPERTY_SEARCH_RUNS' => null, ]); } } From 3cef3448c18772b217570f7408294e8588b1e5b4 Mon Sep 17 00:00:00 2001 From: "v.razuvaev" Date: Sun, 20 Sep 2026 04:15:26 +0300 Subject: [PATCH 2/2] Release 0.10.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a7b0b7..43850c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.10.0 — 2026-09-20 - **Added:** the core 0.12 knobs on the chain — `exhaustive()` / `exhaustiveBudget()` (walk the parameter domain instead of sampling it),