diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 5056e043df..7ad63ea77e 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2025-12-09 18:02:29 UTC using RuboCop version 1.71.2. +# on 2025-12-15 17:20:38 UTC using RuboCop version 1.71.2. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -11,6 +11,7 @@ # TODO - [LH] -> Jul '24 - 369 files inspected, 661 offenses detected, 98 offenses autocorrectable # TODO - [LH] -> Jan '25 (Updated deps and v10 prep) - 369 files inspected, 704 offenses detected, 112 offenses autocorrectable # TODO - [LH] -> Mar '25 (v10 prep) - 370 files inspected, 721 offenses detected, 116 offenses autocorrectable +# TODO - [LH] -> Dec '25 - 375 files inspected, 713 offenses detected, 109 offenses autocorrectable # TODO - [LH] -> Dec '26 (query prep) - 378 files inspected, 729 offenses detected, 109 offenses autocorrectable # Offense count: 1 @@ -56,12 +57,12 @@ Metrics/AbcSize: # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode. # AllowedMethods: refine Metrics/BlockLength: - Max: 52 + Max: 53 # Offense count: 14 # Configuration parameters: CountComments, CountAsOne. Metrics/ClassLength: - Max: 515 + Max: 516 # Offense count: 10 # Configuration parameters: AllowedMethods, AllowedPatterns. @@ -71,7 +72,7 @@ Metrics/CyclomaticComplexity: # Offense count: 83 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. Metrics/MethodLength: - Max: 64 + Max: 65 # Offense count: 15 # Configuration parameters: CountComments, CountAsOne. diff --git a/CHANGELOG.md b/CHANGELOG.md index abe80e82ed..6f2cbad7c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo ## [Unreleased] ### Added +- Added a new option for running order `--reverse` which will run the scenarios in reverse order ([#1807](https://github.com/cucumber/cucumber-ruby/pull/1807) [luke-hill](https://github.com/luke-hill)) - A first initial iteration of the new `cucumber-query` structure ([#1801](https://github.com/cucumber/cucumber-ruby/pull/1801) [luke-hill](https://github.com/luke-hill)) > This will be used for the migration of all existing formatters - becoming the building blocks for the future of cucumber formatters > which will begin being migrated in the start of 2026 diff --git a/features/docs/cli/ordering.feature b/features/docs/cli/ordering.feature new file mode 100644 index 0000000000..1c49a98cbd --- /dev/null +++ b/features/docs/cli/ordering.feature @@ -0,0 +1,244 @@ +Feature: Ordering + + Cucumber can run scenarios in different orders. By default, scenarios are run in the order they + appear in the feature files. Use the `--order random` switch to run scenarios in random order. + + You can also run cucumber in a reverse order using `--order reverse`. + + Using different ordering can help you detect situations where you have state + leaking between scenarios, which can cause flickering or fragile tests. + + If you do find a random run that exposes dependencies between your tests, + you can reproduce that run by using the seed that's printed at the end of + the test run. + + For a given seed, the order of scenarios is constant, i.e. if step A runs + before step B, it will always run before step B even if other steps are + skipped. + + Background: + Given a file named "features/bad_practice_part_1.feature" with: + """ + Feature: Bad practice, part 1 + + Scenario: Set state + Given I set some state + + Scenario: Depend on state from a preceding scenario + When I depend on the state + """ + And a file named "features/bad_practice_part_2.feature" with: + """ + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + """ + And a file named "features/unrelated.feature" with: + """ + Feature: Unrelated + + @skipme + Scenario: Do something unrelated + When I do something + """ + And a file named "features/step_definitions/steps.rb" with: + """ + Given('I set some state') do + $global_state = 'set' + end + + Given('I depend on the state') do + raise 'I expect the state to be set!' unless $global_state == 'set' + end + + Given('I do something') do + end + """ + + Scenario: Run scenarios in order + When I run `cucumber` + Then it should pass + And the stdout should contain exactly: + """ + Feature: Bad practice, part 1 + + Scenario: Set state # features/bad_practice_part_1.feature:3 + Given I set some state # features/step_definitions/steps.rb:1 + + Scenario: Depend on state from a preceding scenario # features/bad_practice_part_1.feature:6 + When I depend on the state # features/step_definitions/steps.rb:5 + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature # features/bad_practice_part_2.feature:3 + When I depend on the state # features/step_definitions/steps.rb:5 + + Feature: Unrelated + + @skipme + Scenario: Do something unrelated # features/unrelated.feature:4 + When I do something # features/step_definitions/steps.rb:9 + + 4 scenarios (4 passed) + 4 steps (4 passed) + 0m0.012s + """ + + @global_state + Scenario: Run scenarios randomized + When I run `cucumber --order random:41544 -q` + Then it should fail + And the stdout should contain exactly: + """ + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Feature: Unrelated + + @skipme + Scenario: Do something unrelated + When I do something + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_1.feature:6 + cucumber features/bad_practice_part_2.feature:3 + + 4 scenarios (2 failed, 2 passed) + 4 steps (2 failed, 2 passed) + + Randomized with seed 41544 + """ + + @force_legacy_loader + Scenario: Rerun scenarios randomized + When I run `cucumber --order random --format summary` + And I rerun the previous command with the same seed + Then the output of both commands should be the same + + @global_state + Scenario: Run scenarios randomized with some skipped + When I run `cucumber --tags "not @skipme" --order random:41544 -q` + Then it should fail with exactly: + """ + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_1.feature:6 + cucumber features/bad_practice_part_2.feature:3 + + 3 scenarios (2 failed, 1 passed) + 3 steps (2 failed, 1 passed) + + Randomized with seed 41544 + + """ + + @global_state + Scenario: Run scenarios in reverse order + When I run `cucumber --order reverse -q` + Then it should fail + And the stdout should contain exactly: + """ + Feature: Unrelated + + @skipme + Scenario: Do something unrelated + When I do something + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_2.feature:3 + cucumber features/bad_practice_part_1.feature:6 + + 4 scenarios (2 failed, 2 passed) + 4 steps (2 failed, 2 passed) + """ + + @global_state + Scenario: Run scenarios in reverse order with some skipped + When I run `cucumber --tags "not @skipme" --order reverse -q` + Then it should fail + And the stdout should contain exactly: + """ + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_2.feature:3 + cucumber features/bad_practice_part_1.feature:6 + + 3 scenarios (2 failed, 1 passed) + 3 steps (2 failed, 1 passed) + """ diff --git a/features/docs/cli/randomize.feature b/features/docs/cli/randomize.feature deleted file mode 100644 index f810a774e6..0000000000 --- a/features/docs/cli/randomize.feature +++ /dev/null @@ -1,144 +0,0 @@ -Feature: Randomize - - Use the `--order random` switch to run scenarios in random order. - - This is especially helpful for detecting situations where you have state - leaking between scenarios, which can cause flickering or fragile tests. - - If you do find a random run that exposes dependencies between your tests, - you can reproduce that run by using the seed that's printed at the end of - the test run. - - For a given seed, the order of scenarios is constant, i.e. if step A runs - before step B, it will always run before step B even if other steps are - skipped. - - Background: - Given a file named "features/bad_practice_part_1.feature" with: - """ - Feature: Bad practice, part 1 - - Scenario: Set state - Given I set some state - - Scenario: Depend on state from a preceding scenario - When I depend on the state - """ - And a file named "features/bad_practice_part_2.feature" with: - """ - Feature: Bad practice, part 2 - - Scenario: Depend on state from a preceding feature - When I depend on the state - """ - And a file named "features/unrelated.feature" with: - """ - Feature: Unrelated - - @skipme - Scenario: Do something unrelated - When I do something - """ - And a file named "features/step_definitions/steps.rb" with: - """ - Given(/^I set some state$/) do - $global_state = "set" - end - - Given(/^I depend on the state$/) do - raise "I expect the state to be set!" unless $global_state == "set" - end - - Given(/^I do something$/) do - end - """ - - Scenario: Run scenarios in order - When I run `cucumber` - Then it should pass - - @global_state - Scenario: Run scenarios randomized - When I run `cucumber --order random:41544 -q` - Then it should fail - And the stdout should contain exactly: - """ - Feature: Bad practice, part 1 - - Scenario: Depend on state from a preceding scenario - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_1.feature:7:in `I depend on the state' - - Feature: Unrelated - - @skipme - Scenario: Do something unrelated - When I do something - - Feature: Bad practice, part 2 - - Scenario: Depend on state from a preceding feature - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_2.feature:4:in `I depend on the state' - - Feature: Bad practice, part 1 - - Scenario: Set state - Given I set some state - - Failing Scenarios: - cucumber features/bad_practice_part_1.feature:6 - cucumber features/bad_practice_part_2.feature:3 - - 4 scenarios (2 failed, 2 passed) - 4 steps (2 failed, 2 passed) - - Randomized with seed 41544 - """ - - @force_legacy_loader - Scenario: Rerun scenarios randomized - When I run `cucumber --order random --format summary` - And I rerun the previous command with the same seed - Then the output of both commands should be the same - - @global_state - Scenario: Run scenarios randomized with some skipped - When I run `cucumber --tags "not @skipme" --order random:41544 -q` - Then it should fail with exactly: - """ - Feature: Bad practice, part 1 - - Scenario: Depend on state from a preceding scenario - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_1.feature:7:in `I depend on the state' - - Feature: Bad practice, part 2 - - Scenario: Depend on state from a preceding feature - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_2.feature:4:in `I depend on the state' - - Feature: Bad practice, part 1 - - Scenario: Set state - Given I set some state - - Failing Scenarios: - cucumber features/bad_practice_part_1.feature:6 - cucumber features/bad_practice_part_2.feature:3 - - 3 scenarios (2 failed, 1 passed) - 3 steps (2 failed, 1 passed) - - Randomized with seed 41544 - - """ diff --git a/lib/cucumber/cli/options.rb b/lib/cucumber/cli/options.rb index 3e9104567b..218112e54d 100644 --- a/lib/cucumber/cli/options.rb +++ b/lib/cucumber/cli/options.rb @@ -63,7 +63,7 @@ class Options PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, RETRY_TOTAL_FLAG, '-l', '--lines', '--port', '-I', '--snippet-type' ].freeze - ORDER_TYPES = %w[defined random].freeze + ORDER_TYPES = %w[defined random reverse].freeze TAG_LIMIT_MATCHER = /(?@\w+):(?\d+)/x.freeze def self.parse(args, out_stream, error_stream, options = {}) @@ -146,6 +146,7 @@ def parse!(args) *<<~TEXT.split("\n")) do |order| [defined] Run scenarios in the order they were defined (default). [random] Shuffle scenarios before running. + [reverse] Run scenarios in the opposite order to which they were defined. Specify SEED to reproduce the shuffling from a previous run. e.g. --order random:5738 TEXT diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb index 7ce2c73a03..db394caec1 100644 --- a/lib/cucumber/configuration.rb +++ b/lib/cucumber/configuration.rb @@ -54,6 +54,10 @@ def randomize? @options[:order] == 'random' end + def reverse_order? + @options[:order] == 'reverse' + end + def seed @options[:seed] end diff --git a/lib/cucumber/filters.rb b/lib/cucumber/filters.rb index 6ca9c5d023..d14c38bdd4 100644 --- a/lib/cucumber/filters.rb +++ b/lib/cucumber/filters.rb @@ -9,6 +9,7 @@ require 'cucumber/filters/prepare_world' require 'cucumber/filters/quit' require 'cucumber/filters/randomizer' +require 'cucumber/filters/reverser' require 'cucumber/filters/retry' require 'cucumber/filters/tag_limits' require 'cucumber/filters/broadcast_test_case_ready_event' diff --git a/lib/cucumber/filters/randomizer.rb b/lib/cucumber/filters/randomizer.rb index 92cf6d5884..f2fd517c73 100644 --- a/lib/cucumber/filters/randomizer.rb +++ b/lib/cucumber/filters/randomizer.rb @@ -6,6 +6,9 @@ module Cucumber module Filters # Batches up all test cases, randomizes them, and then sends them on class Randomizer + attr_reader :seed + private :seed + def initialize(seed, receiver = nil) @receiver = receiver @test_cases = [] @@ -26,7 +29,7 @@ def done end def with_receiver(receiver) - self.class.new(@seed, receiver) + self.class.new(seed, receiver) end private @@ -34,12 +37,9 @@ def with_receiver(receiver) def shuffled_test_cases digester = Digest::SHA2.new(256) @test_cases.map.with_index - .sort_by { |_, index| digester.digest((@seed + index).to_s) } + .sort_by { |_, index| digester.digest((seed + index).to_s) } .map { |test_case, _| test_case } end - - attr_reader :seed - private :seed end end end diff --git a/lib/cucumber/filters/reverser.rb b/lib/cucumber/filters/reverser.rb new file mode 100644 index 0000000000..6bc21c3f47 --- /dev/null +++ b/lib/cucumber/filters/reverser.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'digest/sha2' + +module Cucumber + module Filters + # Reverses the order of test cases + class Reverser + attr_reader :seed + private :seed + + def initialize(receiver = nil) + @receiver = receiver + @test_cases = [] + end + + def test_case(test_case) + @test_cases << test_case + self + end + + def done + reversed_test_cases.each do |test_case| + test_case.describe_to(@receiver) + end + @receiver.done + self + end + + def with_receiver(receiver) + self.class.new(receiver) + end + + private + + def reversed_test_cases + @test_cases.reverse + end + end + end +end diff --git a/lib/cucumber/runtime.rb b/lib/cucumber/runtime.rb index da05e6f211..3315fd171d 100644 --- a/lib/cucumber/runtime.rb +++ b/lib/cucumber/runtime.rb @@ -244,6 +244,7 @@ def filters filters << Cucumber::Core::Test::NameFilter.new(name_regexps) filters << Cucumber::Core::Test::LocationsFilter.new(filespecs.locations) filters << Filters::Randomizer.new(@configuration.seed) if @configuration.randomize? + filters << Filters::Reverser.new if @configuration.reverse_order? # TODO: can we just use Glue::RegistryAndMore's step definitions directly? step_match_search = StepMatchSearch.new(@support_code.registry.method(:step_matches), @configuration) filters << Filters::ActivateSteps.new(step_match_search, @configuration)