diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c943ac72..263e6e3d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,11 +16,11 @@ jobs: fail-fast: false matrix: ruby: - - '2.7' - - '3.0' - '3.1' - '3.2' - - '3.3.0' + - '3.3' + - '3.4' + - '4.0' - 'truffleruby' steps: - uses: actions/checkout@v2 @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] + python: [ '3.8', '3.9', '3.10', '3.11' ] services: redis: @@ -60,7 +60,7 @@ jobs: sudo add-apt-repository -y ppa:deadsnakes/ppa sudo apt-get -qq update sudo apt-get install -y python${{ matrix.python }} python${{ matrix.python }}-distutils - sudo pip install autopep8 + sudo pip install autopep8 || true - name: Run Python tests run: | bin/before-install diff --git a/bin/before-install b/bin/before-install index 6da4c556..8a11d08b 100755 --- a/bin/before-install +++ b/bin/before-install @@ -10,7 +10,6 @@ end case ENV['SUITE'] when 'python' Dir.chdir('python/') do - run('sudo', 'pip', 'install', '-U', 'pip') run('sudo', 'pip', 'install', 'setuptools==68.0.0') run('pip', '--version') run('sudo', 'make', 'install') diff --git a/python/Makefile b/python/Makefile index 297d7bba..8a4e4624 100644 --- a/python/Makefile +++ b/python/Makefile @@ -26,7 +26,7 @@ autolint: autopep8 lint run_tests: clean tox -e ${TOX_ENV} -- --durations=10 -vv tests -test: autopep8 run_tests +test: run_tests install: pip install --ignore-installed six -e .[test] diff --git a/python/ciqueue/distributed.py b/python/ciqueue/distributed.py index 30a2cb77..e56d342e 100644 --- a/python/ciqueue/distributed.py +++ b/python/ciqueue/distributed.py @@ -90,8 +90,14 @@ def shutdown(self): def acknowledge(self, test): return self._eval_script( 'acknowledge', - keys=[self.key('running'), self.key('processed'), self.key('owners')], - args=[test], + keys=[ + self.key('running'), + self.key('processed'), + self.key('owners'), + self.key('error-reports'), + self.key('requeued-by'), + ], + args=[test, test, '', 0], ) == 1 def requeue(self, test, offset=42): @@ -107,8 +113,10 @@ def requeue(self, test, offset=42): self.key('running'), self.key('worker', self.worker_id, 'queue'), self.key('owners'), + self.key('error-reports'), + self.key('requeued-by'), ], - args=[self.max_requeues, self.global_max_requeues, test, offset], + args=[self.max_requeues, self.global_max_requeues, test, test, offset, 0], ) == 1 def retry_queue(self): @@ -173,9 +181,12 @@ def _try_to_reserve_test(self): self.key('processed'), self.key('worker', self.worker_id, 'queue'), self.key('owners'), + self.key('requeued-by'), + self.key('workers'), ], args=[ time.time(), + 42, ], ) diff --git a/python/tests/test_pytest.py b/python/tests/test_pytest.py index 895565ec..c3a7e0dc 100644 --- a/python/tests/test_pytest.py +++ b/python/tests/test_pytest.py @@ -13,7 +13,7 @@ def change_test_dir(request, monkeypatch): def expected_messages(output): - assert '= 4 failed, 2 passed, 1 skipped, 1 xpassed, 6 errors in' in output, output + assert re.search(r'= 4 failed, 2 passed, 1 skipped, 1 xpassed, (1 warning, )?6 errors in', output), output assert re.search(r':\d+: skipping test message', output) is not None, \ "did not find 'skipping test message' in output" @@ -64,7 +64,7 @@ def test_retries_and_junit_xml(self, tmpdir): .format(queue, xml_file, filename) output = check_output(cmd.format(queue, filename)) - assert '= 4 failed, 2 passed, 4 skipped, 1 xpassed, 6 errors in' in output, output + assert re.search(r'= 4 failed, 2 passed, 4 skipped, 1 xpassed, (1 warning, )?6 errors in', output), output assert ('integrations/pytest/test_all.py:27: skipping test message' in output or 'integrations/pytest/test_all.py:28: skipping test message' in output), output assert ' WILL_RETRY ' in output, output diff --git a/ruby/.ruby-version b/ruby/.ruby-version index 15a27998..5186d070 100644 --- a/ruby/.ruby-version +++ b/ruby/.ruby-version @@ -1 +1 @@ -3.3.0 +4.0 diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock index 5c450137..04a47113 100644 --- a/ruby/Gemfile.lock +++ b/ruby/Gemfile.lock @@ -7,89 +7,99 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (7.1.3.2) + activesupport (8.1.3) base64 bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) ansi (1.5.0) - ast (2.4.2) - base64 (0.2.0) - bigdecimal (3.1.7) - builder (3.2.4) - concurrent-ruby (1.2.3) - connection_pool (2.4.1) - diff-lcs (1.5.1) - docile (1.4.0) - drb (2.2.1) - i18n (1.14.4) + ast (2.4.3) + base64 (0.3.0) + benchmark (0.5.0) + bigdecimal (4.0.1) + builder (3.3.0) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) + diff-lcs (1.6.2) + docile (1.4.1) + drb (2.2.3) + i18n (1.14.8) concurrent-ruby (~> 1.0) - json (2.7.1) - language_server-protocol (3.17.0.3) - logger (1.6.1) - minitest (5.22.3) - minitest-reporters (1.6.1) + json (2.19.3) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + logger (1.7.0) + minitest (5.27.0) + minitest-reporters (1.7.1) ansi builder minitest (>= 5.0) ruby-progressbar - msgpack (1.7.2) - mutex_m (0.2.0) - parallel (1.24.0) - parser (3.3.0.5) + msgpack (1.8.0) + parallel (1.27.0) + parser (3.3.10.2) ast (~> 2.4.1) racc - racc (1.7.3) + prism (1.9.0) + racc (1.8.1) rainbow (3.1.1) - rake (13.1.0) - redis (5.1.0) - redis-client (>= 0.17.0) - redis-client (0.21.1) + rake (13.3.1) + redis (5.4.1) + redis-client (>= 0.22.0) + redis-client (0.28.0) connection_pool - regexp_parser (2.9.0) - rexml (3.3.8) - rspec (3.13.0) + regexp_parser (2.11.3) + rexml (3.4.4) + rspec (3.13.2) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.6) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.8) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.1) - rubocop (1.62.1) + rspec-support (3.13.7) + rubocop (1.86.0) json (~> 2.3) - language_server-protocol (>= 3.17.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.1) + parser (>= 3.3.7.2) + prism (~> 1.7) ruby-progressbar (1.13.0) + securerandom (0.4.1) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) + simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) - snappy (0.4.0) + snappy (0.5.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + uri (1.1.1) PLATFORMS arm64-darwin-23 @@ -97,6 +107,7 @@ PLATFORMS DEPENDENCIES activesupport + benchmark bundler ci-queue! minitest (~> 5.11) @@ -104,6 +115,7 @@ DEPENDENCIES msgpack rake redis + rexml rspec (~> 3.10) rubocop simplecov (~> 0.12) diff --git a/ruby/ci-queue.gemspec b/ruby/ci-queue.gemspec index dd080b4a..6b2c7387 100644 --- a/ruby/ci-queue.gemspec +++ b/ruby/ci-queue.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.homepage = 'https://github.com/Shopify/ci-queue' spec.license = 'MIT' - spec.required_ruby_version = '>= 2.7' + spec.required_ruby_version = '>= 3.1' spec.files = lua_scripts + `git ls-files -z`.split("\x0").reject do |f| f.match(%r{^(test|spec|features)/}) @@ -43,5 +43,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'snappy' spec.add_development_dependency 'msgpack' + spec.add_development_dependency 'benchmark' + spec.add_development_dependency 'rexml' spec.add_development_dependency 'rubocop' end diff --git a/ruby/lib/minitest/queue/test_data.rb b/ruby/lib/minitest/queue/test_data.rb index c33dbb9b..0043d220 100644 --- a/ruby/lib/minitest/queue/test_data.rb +++ b/ruby/lib/minitest/queue/test_data.rb @@ -138,7 +138,7 @@ def error_location(exception) @error_location ||= begin last_before_assertion = '' backtrace_for(exception).reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ + break if s =~ /in [`'](?:[\w:]*[#.])?(assert|refute|flunk|pass|fail|raise|must|wont)/ last_before_assertion = s end diff --git a/ruby/test/integration/grind_redis_test.rb b/ruby/test/integration/grind_redis_test.rb index b99033aa..48ed362b 100644 --- a/ruby/test/integration/grind_redis_test.rb +++ b/ruby/test/integration/grind_redis_test.rb @@ -47,7 +47,7 @@ def test_grind_command_success all tests passed every time, grinding did not uncover any flakiness EOS assert_equal expected.strip, output - assert_empty err + assert_empty filter_deprecation_warnings(err) end def test_grind_command_runs_tests @@ -91,7 +91,7 @@ def test_grind_command_runs_tests \tExpected false to be truthy. \t test/dummy_test.rb:18:in `test_flaky' EOS - assert_empty err + assert_empty filter_deprecation_warnings(err) assert_equal expected.strip, output end @@ -149,7 +149,7 @@ def test_grind_max_time run_count = runs_line.scan(/\w+/).last.to_i - assert_empty err + assert_empty filter_deprecation_warnings(err) assert run_count < grind_count end @@ -205,7 +205,7 @@ def test_can_grind_multiple_things \t test/dummy_test.rb:10:in `test_bar' EOS assert_equal expected.strip, output - assert_empty err + assert_empty filter_deprecation_warnings(err) end def test_grind_max_test_duration_passing @@ -248,7 +248,7 @@ def test_grind_max_test_duration_passing The 50th of test execution time is within 1000.0 milliseconds. EOS assert_equal expected.strip, output - assert_empty err + assert_empty filter_deprecation_warnings(err) end def test_grind_max_test_duration_failing @@ -293,7 +293,7 @@ def test_grind_max_test_duration_failing test_flaky_passes: EOS assert output.start_with?(expected.strip) - assert_empty err + assert_empty filter_deprecation_warnings(err) end def test_grind_max_test_duration_percentile_outside_range diff --git a/ruby/test/integration/minitest_bisect_test.rb b/ruby/test/integration/minitest_bisect_test.rb index 6af1cd15..f317f3e7 100644 --- a/ruby/test/integration/minitest_bisect_test.rb +++ b/ruby/test/integration/minitest_bisect_test.rb @@ -15,7 +15,7 @@ def test_bisect run_bisect('log/leaky_test_order.log', 'LeakyTest#test_sensible_to_leak') end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation LeakyTest#test_sensible_to_leak PASS @@ -105,7 +105,7 @@ def test_inconclusive run_bisect('log/unconclusive_test_order.log', 'LeakyTest#test_sensible_to_leak') end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation LeakyTest#test_sensible_to_leak PASS @@ -182,7 +182,7 @@ def test_failing_test_is_the_first_entry_in_the_test_order run_bisect('log/unconclusive_test_order.log', 'LeakyTest#test_useless_0') end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation LeakyTest#test_useless_0 PASS @@ -197,7 +197,7 @@ def test_broken_tests_which_are_not_evaluated_are_ignored run_bisect('log/leaky_with_broken_test_order.log', 'LeakyTest#test_sensible_to_leak') end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation LeakyTest#test_sensible_to_leak PASS @@ -274,7 +274,7 @@ def test_broken run_bisect('log/broken_test_order.log', 'LeakyTest#test_broken_test') end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation LeakyTest#test_broken_test FAIL @@ -291,7 +291,7 @@ def test_failing_test_is_not_present run_bisect('log/leaky_test_order.log', 'LeakyTestDoesNotExist#test_sensible_to_leak') end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation ^^^ +++ @@ -305,7 +305,7 @@ def test_failing_test_is_not_present private def normalize(output) - rewrite_paths(freeze_seed(freeze_timing(decolorize_output(output)))) + normalize_backtrace(rewrite_paths(freeze_seed(freeze_timing(decolorize_output(output))))) end def run_bisect(test_order_file, failing_test) diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb index 33258dcf..a075fd75 100644 --- a/ruby/test/integration/minitest_redis_test.rb +++ b/ruby/test/integration/minitest_redis_test.rb @@ -42,7 +42,7 @@ def test_default_reporter ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) assert_match(/Expected false to be truthy/, normalize(out)) # failure output result = normalize(out.lines.last.strip) assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result @@ -71,7 +71,7 @@ def test_lost_test_with_heartbeat_monitor end.each(&:join) end - assert_empty err + assert_empty filter_deprecation_warnings(err) Tempfile.open('warnings') do |warnings_file| out, err = capture_subprocess_io do @@ -86,7 +86,7 @@ def test_lost_test_with_heartbeat_monitor ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) result = normalize(out.lines[1].strip) # lost_test.rb test_foo has no assertions (only sleep) assert_equal "Ran 1 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)", result @@ -119,7 +119,7 @@ def test_lazy_loading_streaming threads.each(&:join) end - assert_empty err + assert_empty filter_deprecation_warnings(err) out, err = capture_subprocess_io do system( @@ -131,7 +131,7 @@ def test_lazy_loading_streaming ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) result = normalize(out.lines[1].strip) assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)', result end @@ -174,7 +174,7 @@ def test_lazy_loading_with_test_files_option threads.each(&:join) end - assert_empty err + assert_empty filter_deprecation_warnings(err) # Verify the non-leader actually entered queue.poll and processed tests. # The leader may process 0 tests if the non-leader is fast enough to drain @@ -195,7 +195,7 @@ def test_lazy_loading_with_test_files_option ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) result = normalize(out.lines[1].strip) assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)', result end @@ -235,7 +235,7 @@ def test_lazy_loading_dynamic_test_methods threads.each(&:join) end - assert_empty err + assert_empty filter_deprecation_warnings(err) out, err = capture_subprocess_io do system( @@ -247,7 +247,7 @@ def test_lazy_loading_dynamic_test_methods ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) # 1 static + 3 dynamic variants = 4 tests result = normalize(out.lines[1].strip) assert_equal 'Ran 4 tests, 4 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)', result @@ -274,7 +274,7 @@ def test_worker_profile_in_report ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) out, err = capture_subprocess_io do system( @@ -287,7 +287,7 @@ def test_worker_profile_in_report ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) assert_includes out, 'Worker profile summary' assert_includes out, 'leader' assert_includes out, 'Wall Clock' @@ -312,7 +312,7 @@ def test_verbose_reporter ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) assert_match(/ATest#test_foo \d+\.\d+ = S/, normalize(out)) # verbose test ouptut result = normalize(out.lines.last.strip) assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result @@ -339,7 +339,7 @@ def test_debug_log end assert_includes File.read(log_file.path), 'INFO -- : Finished \'["exists", "build:1:worker:1:queue"]\': 0' - assert_empty err + assert_empty filter_deprecation_warnings(err) result = normalize(out.lines.last.strip) assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result end @@ -363,7 +363,7 @@ def test_buildkite_output ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) assert_match(/^\^{3} \+{3}$/m, normalize(out)) # reopen failed step output = normalize(out.lines.last.strip) assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output @@ -387,7 +387,7 @@ def test_custom_requeue ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal '--- Ran 3 tests, 0 assertions, 0 failures, 2 errors, 0 skips, 1 requeues in X.XXs', output end @@ -413,7 +413,7 @@ def test_max_test_failed refute_predicate $?, :success? assert_equal 1, $?.exitstatus - assert_equal 'This worker is exiting early because too many failed tests were encountered.', err.chomp + assert_equal 'This worker is exiting early because too many failed tests were encountered.', filter_deprecation_warnings(err).chomp output = normalize(out.lines.last.strip) assert_equal 'Ran 47 tests, 47 assertions, 3 failures, 0 errors, 0 skips, 44 requeues in X.XXs', output @@ -432,7 +432,7 @@ def test_max_test_failed refute_predicate $?, :success? assert_equal 44, $?.exitstatus - assert_empty err + assert_empty filter_deprecation_warnings(err) expected = <<~EXPECTED Waiting for workers to complete Requeued 44 tests @@ -468,7 +468,7 @@ def test_all_workers_died refute_predicate $?, :success? assert_equal 40, $?.exitstatus - assert_empty err + assert_empty filter_deprecation_warnings(err) expected = <<~EXPECTED Waiting for workers to complete No leader was elected. This typically means no worker was able to start. Were there any errors during application boot? @@ -494,7 +494,7 @@ def test_circuit_breaker ) end - assert_equal "This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state.\n", err + assert_equal "This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state.\n", filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 3 tests, 3 assertions, 0 failures, 0 errors, 0 skips, 3 requeues in X.XXs', output end @@ -516,7 +516,7 @@ def test_redis_runner ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output @@ -536,7 +536,7 @@ def test_redis_runner ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 6 tests, 4 assertions, 2 failures, 1 errors, 0 skips, 3 requeues in X.XXs', output end @@ -557,7 +557,7 @@ def test_retry_success chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output @@ -576,7 +576,7 @@ def test_retry_success chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'All tests were ran already', output end @@ -597,7 +597,7 @@ def test_automatic_retry chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 200 tests, 200 assertions, 100 failures, 0 errors, 0 skips, 100 requeues in X.XXs', output @@ -617,7 +617,7 @@ def test_automatic_retry chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'All tests were ran already', output end @@ -638,7 +638,7 @@ def test_retry_fails_when_test_run_is_expired chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output @@ -661,7 +661,7 @@ def test_retry_fails_when_test_run_is_expired chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal "The test run is too old and can't be retried", output end @@ -681,7 +681,7 @@ def test_retry_report chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 100 tests, 100 assertions, 100 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output @@ -696,7 +696,7 @@ def test_retry_report chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expect = 'Ran 100 tests, 100 assertions, 100 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)' assert_equal expect, normalize(out.strip.lines[1].strip) @@ -737,7 +737,7 @@ def test_retry_report chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'All tests were ran already', output @@ -752,7 +752,7 @@ def test_retry_report chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expect = 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)' assert_equal expect, normalize(out.strip.lines[1].strip) end @@ -774,7 +774,7 @@ def test_down_redis chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 0 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output end @@ -797,7 +797,7 @@ def test_test_data_reporter chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 9 tests, 6 assertions, 1 failures, 1 errors, 1 skips, 2 requeues in X.XXs', output @@ -899,7 +899,7 @@ def test_junit_reporter chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) assert_equal 'Ran 9 tests, 6 assertions, 1 failures, 1 errors, 1 skips, 2 requeues in X.XXs', output @@ -1110,7 +1110,7 @@ def test_redis_reporter chdir: 'test/fixtures/', ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last.strip) # 8 = sum of test.assertions from Minitest (skip counts as 1 in some versions) assert_equal 'Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output @@ -1134,7 +1134,7 @@ def test_redis_reporter assert_equal "Atest#test_bar", content[0]["test"] assert_equal 2, content[0]["timeout"] - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out) expected_output = <<~END @@ -1210,7 +1210,7 @@ def test_utf8_tests_and_marshal ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) output = normalize(out.lines.last) assert_equal <<~END, output Ran 1 tests, 1 assertions, 1 failures, 0 errors, 0 skips, 0 requeues in X.XXs @@ -1257,7 +1257,7 @@ def test_application_error private def normalize_xml(output) - freeze_xml_timing(rewrite_paths(output)) + normalize_backtrace(freeze_xml_timing(rewrite_paths(output))) end end end diff --git a/ruby/test/integration/rspec_redis_test.rb b/ruby/test/integration/rspec_redis_test.rb index 77a2ab5a..20996117 100644 --- a/ruby/test/integration/rspec_redis_test.rb +++ b/ruby/test/integration/rspec_redis_test.rb @@ -32,7 +32,7 @@ def test_redis_runner assert_equal 0, $?.exitstatus end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Worker elected as leader, pushing 3 tests to the queue. @@ -89,7 +89,7 @@ def test_redis_runner_retry ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Worker elected as leader, pushing 3 tests to the queue. @@ -134,7 +134,7 @@ def test_redis_runner_retry ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Found 0 tests to retry, processing the main queue. @@ -166,7 +166,7 @@ def test_retry_report ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = "3 examples, 1 failure" assert_match expected_output, normalize(out) @@ -184,7 +184,7 @@ def test_retry_report ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Waiting for workers to complete 1 error found @@ -240,7 +240,7 @@ def test_retry_report EOS - assert_empty err + assert_empty filter_deprecation_warnings(err) assert_equal expected_retry_output, normalize(out) assert_equal 0, $?.exitstatus @@ -256,7 +256,7 @@ def test_retry_report ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_report_output = strip_heredoc <<-EOS Waiting for workers to complete No errors found @@ -282,7 +282,7 @@ def test_before_suite_errors ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Worker elected as leader, pushing 2 tests to the queue. @@ -326,7 +326,7 @@ def test_report assert_equal 1, $?.exitstatus end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Worker elected as leader, pushing 3 tests to the queue. @@ -369,7 +369,7 @@ def test_report ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS --- Waiting for workers to complete +++ 1 error found @@ -412,7 +412,7 @@ def test_world_wants_to_quit ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS @@ -438,7 +438,7 @@ def test_world_wants_to_quit ) end - assert_empty err + assert_empty filter_deprecation_warnings(err) expected_output = strip_heredoc <<-EOS Worker elected as leader, pushing 1 tests to the queue. @@ -474,7 +474,7 @@ def test_world_wants_to_quit private def normalize(output) - strip_blank_lines(rewrite_paths(freeze_timing(decolorize_output(output)))) + normalize_backtrace(strip_blank_lines(rewrite_paths(freeze_timing(decolorize_output(output))))) end end end diff --git a/ruby/test/minitest/queue/test_time_reporter_test.rb b/ruby/test/minitest/queue/test_time_reporter_test.rb index ed43801b..4c9e147a 100644 --- a/ruby/test/minitest/queue/test_time_reporter_test.rb +++ b/ruby/test/minitest/queue/test_time_reporter_test.rb @@ -19,7 +19,7 @@ def test_report_no_offending_tests \e[32mThe 50th of test execution time is within 0.1 milliseconds.\e[0m EOS assert_equal expected_output, output - assert_empty err + assert_empty filter_deprecation_warnings(err) assert reporter.success? end @@ -32,7 +32,7 @@ def test_report_not_turned_on end assert_equal "", output - assert_empty err + assert_empty filter_deprecation_warnings(err) assert reporter.success? end @@ -51,7 +51,7 @@ def test_report_found_offending_tests some test: 0.03ms EOS assert_equal expected_output, normalize(output) - assert_empty err + assert_empty filter_deprecation_warnings(err) refute reporter.success? end diff --git a/ruby/test/support/output_test_helpers.rb b/ruby/test/support/output_test_helpers.rb index ca968371..c47e997a 100644 --- a/ruby/test/support/output_test_helpers.rb +++ b/ruby/test/support/output_test_helpers.rb @@ -32,7 +32,23 @@ def freeze_xml_timing(output) output.gsub(/time="[\d\-\.e]+"/, 'time="X.XX"').gsub(/timestamp="[\d\-\.e]+"/, 'timestamp="X.XX"') end + def normalize_backtrace(output) + output.to_s.gsub(/in '([^']+)'/) do + "in `#{$1.sub(/\A[A-Z][\w]*(?:::[A-Z][\w]*)*[#.]/, '')}'" + end + end + def normalize(output) - freeze_timing(decolorize_output(output)) + normalize_backtrace(freeze_timing(decolorize_output(output))) + end + + def filter_deprecation_warnings(output) + output.to_s.lines.reject do |line| + line.include?("was loaded from the standard library, but will no longer be part of the default gems") || + line.include?("You can add") && (line.include?("to your Gemfile or gemspec to silence this warning") || line.include?("to your Gemfile or gemspec to fix this error")) || + line.include?("is not part of the default gems since Ruby") || + line.include?("warning: already initialized constant") || + line.include?("warning: previous definition of") + end.join end end