From c261086b58648b193d6aec968ff848b54e6f284a Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 28 Aug 2026 14:46:29 +0100 Subject: [PATCH 1/2] Split slow test jobs - Balance four slow test groups by source size across two shards. - Preserve required checks with independent aggregate jobs. - Share test setup through a reusable workflow. - Start test jobs without waiting for syntax checks. - Fail CI tests when required dependencies are unavailable. - Replace prefix-cache actions with direct installs where still needed. --- .github/workflows/actionlint.yml | 9 - .github/workflows/copilot-setup-steps.yml | 5 +- .github/workflows/release.yml | 6 +- .github/workflows/tests.yml | 282 +++++++++++++++--- Library/Homebrew/dev-cmd/tests.rb | 20 +- .../sorbet/rbi/dsl/homebrew/dev_cmd/tests.rbi | 3 + .../sorbet/rbi/shims/parallel_tests.rbi | 16 + Library/Homebrew/test/dev-cmd/test_spec.rb | 2 +- Library/Homebrew/test/dev-cmd/tests_spec.rb | 38 +++ Library/Homebrew/test/spec_helper.rb | 49 ++- completions/bash/brew | 1 + completions/fish/brew.fish | 1 + completions/zsh/_brew | 5 +- docs/Manpage.md | 4 + manpages/brew.1 | 3 + 15 files changed, 364 insertions(+), 80 deletions(-) create mode 100644 Library/Homebrew/sorbet/rbi/shims/parallel_tests.rbi diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 6f65627541549..33fa7a378b3fe 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -39,16 +39,7 @@ jobs: core: false cask: false - - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 - # Don't cache the prefix for the formulae.brew.sh repository as it won't download its own JSON API files - if: github.repository != 'Homebrew/formulae.brew.sh' - with: - install: actionlint shellcheck zizmor - # Not needed in .github but required for other repositories with multiple uses of cache-homebrew-prefix. - workflow-key: actionlint - - name: Install tools - if: github.repository == 'Homebrew/formulae.brew.sh' run: brew install actionlint shellcheck zizmor - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 331acffc3ca28..e4b75b4ff24b5 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -25,10 +25,7 @@ jobs: - run: brew install-bundler-gems --groups=all # install Homebrew formulae we might need - - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 - with: - install: shellcheck shfmt gh gnu-tar subversion curl - workflow-key: copilot-setup-steps + - run: brew install shellcheck shfmt gh gnu-tar subversion curl # brew tests doesn't like world writable directories - run: sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81b9ba1d8efb7..71bb51108bfd5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,11 +83,7 @@ jobs: cask: false - name: Install Pandoc - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 - with: - install: pandoc - workflow-key: release - uninstall: true + run: brew install pandoc - name: Create and unlock temporary macOS keychain run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9570ac32751cf..d9f498281a364 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,10 +50,7 @@ jobs: run: brew install-bundler-gems --groups=style,typecheck - name: Install actionlint, shellcheck and shfmt - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 - with: - install: actionlint shellcheck shfmt - workflow-key: tests-syntax + run: brew install actionlint shellcheck shfmt - name: Cache style cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -210,7 +207,7 @@ jobs: tests: name: ${{ matrix.name }} - needs: syntax + if: github.repository_owner == 'Homebrew' runs-on: ${{ matrix.runs-on }} timeout-minutes: 30 permissions: @@ -221,29 +218,105 @@ jobs: fail-fast: false matrix: include: - - name: tests (online) + - name: tests (online, 1/2) test-flags: --online --coverage --profile 3 runs-on: ubuntu-latest - - name: tests (generic OS) + online: true + load-only: false + shard: 1 + shard-count: 2 + runtime-cache: tests-online-linux + runtime-log: parallel_runtime_rspec.online.log + - name: tests (online, 2/2) + test-flags: --online --coverage --profile 3 + runs-on: ubuntu-latest + online: true + load-only: false + shard: 2 + shard-count: 2 + runtime-cache: tests-online-linux + runtime-log: parallel_runtime_rspec.online.log + - name: tests (generic OS, 1/2) test-flags: --generic --coverage --profile 3 runs-on: ubuntu-latest - - name: tests (Linux) + online: false + load-only: false + shard: 1 + shard-count: 2 + runtime-cache: tests-generic-linux + runtime-log: parallel_runtime_rspec.generic.log + - name: tests (generic OS, 2/2) + test-flags: --generic --coverage --profile 3 + runs-on: ubuntu-latest + online: false + load-only: false + shard: 2 + shard-count: 2 + runtime-cache: tests-generic-linux + runtime-log: parallel_runtime_rspec.generic.log + - name: tests (Linux, 1/2) test-flags: --coverage --profile 3 runs-on: ubuntu-24.04 - - name: tests (no Sorbet) + online: false + load-only: false + shard: 1 + shard-count: 2 + runtime-cache: tests-linux + runtime-log: parallel_runtime_rspec.log + - name: tests (Linux, 2/2) test-flags: --coverage --profile 3 runs-on: ubuntu-24.04 - - name: tests (macOS) + online: false + load-only: false + shard: 2 + shard-count: 2 + runtime-cache: tests-linux + runtime-log: parallel_runtime_rspec.log + - name: tests (macOS, 1/2) + test-flags: --coverage --profile 3 + runs-on: macos-26 + online: false + load-only: false + shard: 1 + shard-count: 2 + runtime-cache: tests-macos + runtime-log: parallel_runtime_rspec.log + - name: tests (macOS, 2/2) test-flags: --coverage --profile 3 runs-on: macos-26 + online: false + load-only: false + shard: 2 + shard-count: 2 + runtime-cache: tests-macos + runtime-log: parallel_runtime_rspec.log + - name: tests (no Sorbet) + test-flags: --coverage --profile 3 + runs-on: ubuntu-24.04 + online: false + load-only: false + shard: 0 + shard-count: 0 + runtime-cache: tests-no-sorbet-linux + runtime-log: parallel_runtime_rspec.log - name: tests (load-only Linux) test-flags: --load-only runs-on: ubuntu-24.04 - load_only: true + online: false + load-only: true + shard: 0 + shard-count: 0 + runtime-cache: unused + runtime-log: parallel_runtime_rspec.log - name: tests (load-only macOS) test-flags: --load-only runs-on: macos-26 - load_only: true + online: false + load-only: true + shard: 0 + shard-count: 0 + runtime-cache: unused + runtime-log: parallel_runtime_rspec.log steps: - name: Set up Homebrew id: set-up-homebrew @@ -253,7 +326,7 @@ jobs: # homebrew/core is not available by default on GitHub-hosted Ubuntu # runners, and it's expensive to tap it. # Load-only jobs never run examples, so they don't need it either. - core: ${{ runner.os == 'macOS' && matrix.load_only != true }} + core: ${{ runner.os == 'macOS' && matrix.load-only != true }} cask: false - name: Cache Bundler RubyGems @@ -268,31 +341,40 @@ jobs: - name: Install Bundler RubyGems run: brew install-bundler-gems --groups=all - - name: Create parallel test log directory - run: mkdir tests + - name: Untap untrusted preinstalled tap + if: runner.os == 'macOS' && matrix.load-only != true + run: | + if brew tap | grep -qx aws/tap; then + brew untap --force aws/tap + fi - - name: Cache parallel tests log - if: matrix.load_only != true - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: tests - key: ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec-${{ github.sha }} - restore-keys: ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec- + - name: Install Subversion + if: matrix.online || (runner.os == 'macOS' && matrix.load-only != true) + run: brew install subversion - - name: Install brew tests --online dependencies - if: matrix.name == 'tests (online)' - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 - with: - install: curl subversion - workflow-key: tests-tests-online + - name: Create parallel test log directory + run: mkdir tests - - name: Install brew tests macOS dependencies - if: runner.os != 'Linux' && matrix.load_only != true - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 + - name: Restore parallel tests log + if: matrix.load-only != true && (github.event_name == 'pull_request' || matrix.online != true) + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - install: subversion gnupg - workflow-key: tests-tests-macos - uninstall: true + path: tests/${{ matrix.runtime-log }} + key: ${{ matrix.runtime-cache }}-parallel-runtime-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ matrix.runtime-cache }}-parallel-runtime- + ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec- + + - name: Stage parallel tests log + if: matrix.load-only != true && (github.event_name == 'pull_request' || matrix.online != true) + run: | + mkdir -p "$TEST_RUNTIME_PATH" + if [[ -f "tests/$RUNTIME_LOG" ]]; then + cp "tests/$RUNTIME_LOG" "$TEST_RUNTIME_PATH/$RUNTIME_LOG" + fi + env: + RUNTIME_LOG: ${{ matrix.runtime-log }} + TEST_RUNTIME_PATH: ${{ steps.set-up-homebrew.outputs.repository-path }}/Library/Homebrew/tests # brew tests doesn't like world writable directories - name: Cleanup permissions @@ -304,16 +386,54 @@ jobs: run: echo "HOMEBREW_TESTS_NO_SORBET_RUNTIME=1" >> "$GITHUB_ENV" - name: Run brew tests - if: github.event_name == 'pull_request' || matrix.name != 'tests (online)' - run: brew tests ${{ matrix.test-flags }} + if: github.event_name == 'pull_request' || matrix.online != true + run: | + read -ra test_flags <<< "$TEST_FLAGS" + if (( SHARD_COUNT > 0 )); then + test_flags+=("--shard=$SHARD_INDEX/$SHARD_COUNT") + fi + brew tests "${test_flags[@]}" env: HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} # These cannot be queried at the macOS level on GitHub Actions. HOMEBREW_LANGUAGES: en-GB CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + TEST_FLAGS: ${{ matrix.test-flags }} + SHARD_INDEX: ${{ matrix.shard }} + SHARD_COUNT: ${{ matrix.shard-count }} + + - name: Collect parallel tests log + if: matrix.load-only != true && (github.event_name == 'pull_request' || matrix.online != true) && success() + run: | + cache_path="tests/$RUNTIME_LOG" + if (( SHARD_COUNT > 0 )); then + cache_path+=".$SHARD_INDEX" + fi + cp "$TEST_RUNTIME_PATH/$RUNTIME_LOG" "$cache_path" + env: + RUNTIME_LOG: ${{ matrix.runtime-log }} + SHARD_INDEX: ${{ matrix.shard }} + SHARD_COUNT: ${{ matrix.shard-count }} + TEST_RUNTIME_PATH: ${{ steps.set-up-homebrew.outputs.repository-path }}/Library/Homebrew/tests + + - name: Upload parallel tests log + if: matrix.shard-count > 0 && (github.event_name == 'pull_request' || matrix.online != true) && success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.runtime-cache }}-parallel-runtime-${{ matrix.shard }} + path: tests/${{ matrix.runtime-log }}.${{ matrix.shard }} + retention-days: 1 + overwrite: true + + - name: Save parallel tests log + if: matrix.shard-count == 0 && matrix.load-only != true && success() + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: tests/${{ matrix.runtime-log }} + key: ${{ matrix.runtime-cache }}-parallel-runtime-${{ github.run_id }}-${{ github.run_attempt }} - name: Get RSpec JUnit XML filenames - if: matrix.load_only != true + if: matrix.load-only != true id: junit_xml working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }} run: | @@ -322,7 +442,7 @@ jobs: echo "filenames=${filenames%,}" >> "$GITHUB_OUTPUT" - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - if: matrix.load_only != true && (github.event_name == 'pull_request' || matrix.name != 'tests (online)') + if: matrix.load-only != true && (github.event_name == 'pull_request' || matrix.online != true) with: working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }} files: ${{ steps.junit_xml.outputs.filenames }} @@ -331,13 +451,91 @@ jobs: report_type: test_results - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - if: matrix.load_only != true && (github.event_name == 'pull_request' || matrix.name != 'tests (online)') + if: matrix.load-only != true && (github.event_name == 'pull_request' || matrix.online != true) with: working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }} files: Library/Homebrew/test/coverage/coverage.xml disable_search: true token: ${{ secrets.CODECOV_TOKEN }} + - name: Store test result + if: always() && matrix.shard-count > 0 + run: | + mkdir -p test-results + printf '%s\n' "$TEST_RESULT" > "test-results/$SHARD_INDEX" + env: + TEST_RESULT: ${{ job.status }} + SHARD_INDEX: ${{ matrix.shard }} + + - name: Upload test result + if: always() && matrix.shard-count > 0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.runtime-cache }}-test-result-${{ matrix.shard }} + path: test-results/${{ matrix.shard }} + retention-days: 1 + overwrite: true + + tests_result: + name: ${{ matrix.name }} + needs: tests + if: always() + runs-on: ubuntu-slim + strategy: + fail-fast: false + matrix: + include: + - name: tests (online) + online: true + runtime-cache: tests-online-linux + runtime-log: parallel_runtime_rspec.online.log + - name: tests (generic OS) + online: false + runtime-cache: tests-generic-linux + runtime-log: parallel_runtime_rspec.generic.log + - name: tests (Linux) + online: false + runtime-cache: tests-linux + runtime-log: parallel_runtime_rspec.log + - name: tests (macOS) + online: false + runtime-cache: tests-macos + runtime-log: parallel_runtime_rspec.log + steps: + - name: Download test results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ matrix.runtime-cache }}-test-result-* + path: test-results + merge-multiple: true + + - name: Check test results + run: | + test "$(cat test-results/1)" = success + test "$(cat test-results/2)" = success + + - name: Download parallel tests logs + if: success() && (github.event_name == 'pull_request' || matrix.online != true) + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ matrix.runtime-cache }}-parallel-runtime-* + path: tests + merge-multiple: true + + - name: Combine parallel tests logs + if: success() && (github.event_name == 'pull_request' || matrix.online != true) + run: | + cat "tests/$RUNTIME_LOG.1" "tests/$RUNTIME_LOG.2" > "tests/$RUNTIME_LOG" + env: + RUNTIME_LOG: ${{ matrix.runtime-log }} + + - name: Save parallel tests log + if: success() && (github.event_name == 'pull_request' || matrix.online != true) + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: tests/${{ matrix.runtime-log }} + key: ${{ matrix.runtime-cache }}-parallel-runtime-${{ github.run_id }}-${{ github.run_attempt }} + test-bot: name: ${{ matrix.name }} needs: syntax @@ -413,11 +611,7 @@ jobs: core: true cask: false - - uses: Homebrew/actions/cache-homebrew-prefix@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 - with: - install: gnu-tar - workflow-key: test-bot - uninstall: true + - run: brew install gnu-tar - name: Disable Sorbet runtime if: matrix.name == 'test-bot (no Sorbet)' diff --git a/Library/Homebrew/dev-cmd/tests.rb b/Library/Homebrew/dev-cmd/tests.rb index df4607eeab1ae..e3e10dbc22e02 100644 --- a/Library/Homebrew/dev-cmd/tests.rb +++ b/Library/Homebrew/dev-cmd/tests.rb @@ -42,13 +42,15 @@ class Tests < AbstractCommand flag "--only=", description: "Run only `_spec.rb`. Appending `:` will start at a " \ "specific line." + flag "--shard=", + description: "Run only `` of `` test shards." flag "--profile=", description: "Output the slowest tests. When run without `--no-parallel` this will output " \ "the slowest tests for each parallel test process." flag "--seed=", description: "Randomise tests with the specified instead of a random seed." - conflicts "--changed", "--only" + conflicts "--changed", "--only", "--shard" conflicts "--stackprof", "--vernier", "--ruby-prof" named_args :none @@ -60,6 +62,7 @@ def run groups = Homebrew.valid_gem_groups - ["sorbet"] groups << "prof" if args.stackprof? || args.vernier? || args.ruby_prof? Homebrew.install_bundler_gems!(groups:) + require "parallel_tests/rspec/runner" HOMEBREW_LIBRARY_PATH.cd do setup_environment! @@ -146,6 +149,20 @@ def run bundle_args = os_bundle_args(bundle_args) files = os_files(files) + if (shard = args.shard) + match = shard.match(%r{\A(\d+)/(\d+)\z}) + raise UsageError, "Invalid `--shard` argument: #{shard}" unless match + + shard_index = match[1].to_i + shard_count = match[2].to_i + if shard_count <= 1 || shard_index < 1 || shard_index > shard_count + raise UsageError, "Invalid `--shard` argument: #{shard}" + end + + files = ParallelTests::RSpec::Runner.tests_in_groups( + files, shard_count, runtime_log: parallel_rspec_log_path + ).fetch(shard_index - 1) + end if files.blank? if args.changed? opoo "No tests are available to run on this operating system." @@ -311,6 +328,7 @@ def changed_test_files sig { params(bundle_args: T::Array[String]).returns(T::Array[String]) } def os_bundle_args(bundle_args) # for generic tests, remove macOS or Linux specific tests + bundle_args << "--tag" << "~needs_sandbox" non_linux_bundle_args(non_macos_bundle_args(bundle_args)) end diff --git a/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/tests.rbi b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/tests.rbi index 04744abb71a76..44d841ff2241a 100644 --- a/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/tests.rbi +++ b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/tests.rbi @@ -44,6 +44,9 @@ class Homebrew::DevCmd::Tests::Args < Homebrew::CLI::Args sig { returns(T.nilable(String)) } def seed; end + sig { returns(T.nilable(String)) } + def shard; end + sig { returns(T::Boolean) } def stackprof?; end diff --git a/Library/Homebrew/sorbet/rbi/shims/parallel_tests.rbi b/Library/Homebrew/sorbet/rbi/shims/parallel_tests.rbi new file mode 100644 index 0000000000000..cef5c7f99a1b6 --- /dev/null +++ b/Library/Homebrew/sorbet/rbi/shims/parallel_tests.rbi @@ -0,0 +1,16 @@ +# typed: strict + +module ParallelTests + module RSpec + class Runner + sig { + params( + tests: T::Array[String], + num_groups: Integer, + options: T::Hash[Symbol, String], + ).returns(T::Array[T::Array[String]]) + } + def self.tests_in_groups(tests, num_groups, options = {}); end + end + end +end diff --git a/Library/Homebrew/test/dev-cmd/test_spec.rb b/Library/Homebrew/test/dev-cmd/test_spec.rb index 35eeac0994b6e..2c35dcc04ee0d 100644 --- a/Library/Homebrew/test/dev-cmd/test_spec.rb +++ b/Library/Homebrew/test/dev-cmd/test_spec.rb @@ -32,7 +32,7 @@ .and be_a_success end - it "blocks network access when test phase is offline", :integration_test do + it "blocks network access when test phase is offline", :integration_test, :needs_sandbox do skip "Sandbox not available." unless Sandbox.available? skip "Nested sandboxing is not supported." if Sandbox.nested_sandbox? diff --git a/Library/Homebrew/test/dev-cmd/tests_spec.rb b/Library/Homebrew/test/dev-cmd/tests_spec.rb index ac72ace38ebc7..dca023772d3f9 100644 --- a/Library/Homebrew/test/dev-cmd/tests_spec.rb +++ b/Library/Homebrew/test/dev-cmd/tests_spec.rb @@ -3,6 +3,7 @@ require "cmd/shared_examples/args_parse" require "dev-cmd/tests" +require "parallel_tests/rspec/runner" RSpec.describe Homebrew::DevCmd::Tests do it_behaves_like "parseable arguments" @@ -84,6 +85,43 @@ expect(invoked_arguments).to end_with("--", "test/warnings_spec.rb") end end + + context "when sharding tests" do + let(:args) { ["--shard=2/2"] } + + it "runs only the selected shard" do + ENV["CI"] = "1" + allow(Dir).to receive(:glob).with("test/**/*_spec.rb") + .and_return(%w[test/a_spec.rb test/b_spec.rb test/c_spec.rb test/d_spec.rb]) + expect(ParallelTests::RSpec::Runner).to receive(:tests_in_groups) + .with(%w[test/a_spec.rb test/b_spec.rb test/c_spec.rb test/d_spec.rb], 2, + runtime_log: "tests/parallel_runtime_rspec.log") + .and_return([ + %w[test/a_spec.rb test/c_spec.rb], + %w[test/b_spec.rb test/d_spec.rb], + ]) + invoked_arguments = T.let([], T::Array[String]) + allow(tests).to receive(:system) do |*arguments| + invoked_arguments = arguments + system "/usr/bin/true" + end + + tests.run + + expect(invoked_arguments).to end_with("--", "test/b_spec.rb", "test/d_spec.rb") + end + end + end + + describe "#ensure_test_dependency!" do + include Test::Helper::Dependencies + + it "fails when a dependency is missing on CI" do + ENV["CI"] = "1" + + expect { ensure_test_dependency!(false, "Dependency is not installed.") } + .to raise_error(RuntimeError, "Dependency is not installed.") + end end describe "#check_test_environment!", :needs_linux do diff --git a/Library/Homebrew/test/spec_helper.rb b/Library/Homebrew/test/spec_helper.rb index 572027416a5ba..5172d3e76f7ce 100644 --- a/Library/Homebrew/test/spec_helper.rb +++ b/Library/Homebrew/test/spec_helper.rb @@ -68,6 +68,26 @@ HOMEBREW_ALIASES, ].freeze +module Test + module Helper + module Dependencies + extend T::Helpers + + requires_ancestor { RSpec::Core::Pending } + + # Skip missing tools locally, but fail on CI so runner dependency + # regressions cannot silently reduce coverage. + def ensure_test_dependency!(available, message) + return if available + + raise message if ENV["CI"] + + skip message + end + end + end +end + # Make `instance_double` and `class_double` # work when type-checking is active. RSpec::Sorbet.allow_doubles! @@ -145,6 +165,7 @@ config.include(Test::Helper::Formula) config.include(Test::Helper::MkTmpDir) config.include(Test::Helper::Subcommand) + config.include(Test::Helper::Dependencies) config.extend(Test::Helper::TestEach) @@ -166,15 +187,15 @@ end config.before(:each, :needs_java) do - skip "Java is not installed." unless which("java") + ensure_test_dependency!(which("java"), "Java is not installed.") end config.before(:each, :needs_jq) do - skip "jq is not installed." unless which("jq") + ensure_test_dependency!(which("jq"), "jq is not installed.") end config.before(:each, :needs_python) do - skip "Python is not installed." if !which("python3") && !which("python") + ensure_test_dependency!(which("python3") || which("python"), "Python is not installed.") end config.before(:each, :needs_network) do @@ -183,15 +204,15 @@ config.before(:each, :needs_homebrew_core) do core_tap_path = "#{ENV.fetch("HOMEBREW_LIBRARY")}/Taps/homebrew/homebrew-core" - skip "Requires homebrew/core to be tapped." unless Dir.exist?(core_tap_path) + ensure_test_dependency!(Dir.exist?(core_tap_path), "Requires homebrew/core to be tapped.") end config.before(:each, :needs_systemd) do - skip "No SystemD found." unless which("systemctl") + ensure_test_dependency!(which("systemctl"), "No SystemD found.") end config.before(:each, :needs_daemon_manager) do - skip "No LaunchCTL or SystemD found." if !which("systemctl") && !which("launchctl") + ensure_test_dependency!(which("systemctl") || which("launchctl"), "No LaunchCTL or SystemD found.") end config.before do |example| @@ -221,7 +242,7 @@ end config.before(:each, :needs_svn) do - skip svn_client_skip_reason if svn_client_skip_reason + ensure_test_dependency!(false, svn_client_skip_reason) if svn_client_skip_reason if svn_client_path_dirs ENV["PATH"] = PATH.new(ENV.fetch("PATH")).append(svn_client_path_dirs) next @@ -237,7 +258,7 @@ svn_shim = HOMEBREW_SHIMS_PATH/"shared/svn" unless quiet_system svn_shim, "--version" svn_client_skip_reason = "Subversion is not installed." - skip svn_client_skip_reason + ensure_test_dependency!(false, svn_client_skip_reason) end svn_shim_path = Pathname(Utils.popen_read(svn_shim, "--homebrew=print-path").chomp.presence) @@ -246,7 +267,7 @@ svn = which("svn", svn_paths) unless svn svn_client_skip_reason = "svn is not installed." - skip svn_client_skip_reason + ensure_test_dependency!(false, svn_client_skip_reason) end svn_client_path_dirs = [svn.dirname] @@ -254,7 +275,7 @@ end config.before(:each, :needs_svnadmin) do - skip svn_skip_reason if svn_skip_reason + ensure_test_dependency!(false, svn_skip_reason) if svn_skip_reason if svn_path_dirs ENV["PATH"] = PATH.new(ENV.fetch("PATH")).append(svn_path_dirs) next @@ -263,7 +284,7 @@ svnadmin = which("svnadmin") unless svnadmin svn_skip_reason = "svnadmin is not installed." - skip svn_skip_reason + ensure_test_dependency!(false, svn_skip_reason) end svn_path_dirs = [svnadmin.dirname] @@ -272,13 +293,13 @@ config.before(:each, :needs_homebrew_curl) do ENV["HOMEBREW_CURL"] = HOMEBREW_BREWED_CURL_PATH - skip "A `curl` with TLS 1.3 support is required." unless Utils::Curl.curl_supports_tls13? + ensure_test_dependency!(Utils::Curl.curl_supports_tls13?, "A `curl` with TLS 1.3 support is required.") rescue FormulaUnavailableError - skip "No `curl` formula is available." + ensure_test_dependency!(false, "No `curl` formula is available.") end config.before(:each, :needs_unzip) do - skip "Unzip is not installed." unless which("unzip") + ensure_test_dependency!(which("unzip"), "Unzip is not installed.") end config.around do |example| diff --git a/completions/bash/brew b/completions/bash/brew index a791d76d72b9c..41ee8d726a215 100644 --- a/completions/bash/brew +++ b/completions/bash/brew @@ -3103,6 +3103,7 @@ _brew_tests() { --quiet --ruby-prof --seed + --shard --stackprof --verbose --vernier diff --git a/completions/fish/brew.fish b/completions/fish/brew.fish index 79aacdec62f86..9e1037b7fd838 100644 --- a/completions/fish/brew.fish +++ b/completions/fish/brew.fish @@ -1983,6 +1983,7 @@ __fish_brew_complete_arg 'tests' -l profile -d 'Output the n slowest tests. When __fish_brew_complete_arg 'tests' -l quiet -d 'Make some output more quiet' __fish_brew_complete_arg 'tests' -l ruby-prof -d 'Use `ruby-prof` to profile tests' __fish_brew_complete_arg 'tests' -l seed -d 'Randomise tests with the specified value instead of a random seed' +__fish_brew_complete_arg 'tests' -l shard -d 'Run only `index` of `total` test shards' __fish_brew_complete_arg 'tests' -l stackprof -d 'Use `stackprof` to profile tests' __fish_brew_complete_arg 'tests' -l verbose -d 'Make some output more verbose' __fish_brew_complete_arg 'tests' -l vernier -d 'Use `vernier` to profile tests' diff --git a/completions/zsh/_brew b/completions/zsh/_brew index 4bece3b3969f2..7ee50a105cacd 100644 --- a/completions/zsh/_brew +++ b/completions/zsh/_brew @@ -2532,7 +2532,7 @@ _brew_test_bot() { # brew tests _brew_tests() { _arguments \ - '(--only)--changed[Only runs tests on files that were changed from the `main` branch]' \ + '(--only --shard)--changed[Only runs tests on files that were changed from the `main` branch]' \ '--coverage[Generate code coverage reports]' \ '--debug[Enable debugging using `ruby/debug`, or surface the standard `odebug` output]' \ '--fail-fast[Exit early on the first failing test]' \ @@ -2541,11 +2541,12 @@ _brew_tests() { '--load-only[Load each test file independently without running its examples]' \ '--no-parallel[Run tests serially]' \ '--online[Include tests that use the GitHub API and tests that use any of the taps for official external commands]' \ - '(--changed)--only[Run only `test_script_spec.rb`. Appending `:line_number` will start at a specific line]' \ + '(--changed --shard)--only[Run only `test_script_spec.rb`. Appending `:line_number` will start at a specific line]' \ '--profile[Output the n slowest tests. When run without `--no-parallel` this will output the slowest tests for each parallel test process]' \ '--quiet[Make some output more quiet]' \ '(--stackprof --vernier)--ruby-prof[Use `ruby-prof` to profile tests]' \ '--seed[Randomise tests with the specified value instead of a random seed]' \ + '(--changed --only)--shard[Run only `index` of `total` test shards]' \ '(--vernier --ruby-prof)--stackprof[Use `stackprof` to profile tests]' \ '--verbose[Make some output more verbose]' \ '(--stackprof --ruby-prof)--vernier[Use `vernier` to profile tests]' diff --git a/docs/Manpage.md b/docs/Manpage.md index f20717e9bd196..d29139de469c7 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -3925,6 +3925,10 @@ Run Homebrew's unit and integration tests. : Run only `_spec.rb`. Appending `:` will start at a specific line. +`--shard` + +: Run only `` of `` test shards. + `--profile` : Output the *`n`* slowest tests. When run without `--no-parallel` this will diff --git a/manpages/brew.1 b/manpages/brew.1 index 87df419b75cfa..22e9cf0b8fa54 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -2484,6 +2484,9 @@ Use \fBruby\-prof\fP to profile tests\. \fB\-\-only\fP Run only \fB_spec\.rb\fP\&\. Appending \fB:\fP will start at a specific line\. .TP +\fB\-\-shard\fP +Run only \fB\fP of \fB\fP test shards\. +.TP \fB\-\-profile\fP Output the \fIn\fP slowest tests\. When run without \fB\-\-no\-parallel\fP this will output the slowest tests for each parallel test process\. .TP From 3441cd091f6f76e27db3b04c7759a0e5ec0e2401 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 28 Aug 2026 15:32:51 +0100 Subject: [PATCH 2/2] Speed up and expand integration coverage - Test every documented command through an outside-in smoke path. - Record child coverage with Ruby `Coverage` and skip Sorbet loading. - Build installed fixtures directly and combine Formula and Cask paths. --- Library/Homebrew/.simplecov | 48 +------------- Library/Homebrew/dev-cmd/tests.rb | 1 - Library/Homebrew/test/cmd/--caskroom_spec.rb | 15 ++++- Library/Homebrew/test/cmd/--cellar_spec.rb | 8 ++- Library/Homebrew/test/cmd/--env_spec.rb | 5 ++ Library/Homebrew/test/cmd/--prefix_spec.rb | 8 ++- .../Homebrew/test/cmd/--repository_spec.rb | 2 + Library/Homebrew/test/cmd/--taps_spec.rb | 6 ++ Library/Homebrew/test/cmd/alias_spec.rb | 8 +-- Library/Homebrew/test/cmd/analytics_spec.rb | 10 +-- .../Homebrew/test/cmd/as-console-user_spec.rb | 1 + Library/Homebrew/test/cmd/cleanup_spec.rb | 5 +- .../test/cmd/command-not-found-init_spec.rb | 1 + Library/Homebrew/test/cmd/completions_spec.rb | 10 +-- Library/Homebrew/test/cmd/deps_spec.rb | 7 +- Library/Homebrew/test/cmd/desc_spec.rb | 6 +- Library/Homebrew/test/cmd/gist-logs_spec.rb | 8 +++ Library/Homebrew/test/cmd/home_spec.rb | 10 ++- Library/Homebrew/test/cmd/info_spec.rb | 4 +- Library/Homebrew/test/cmd/install_spec.rb | 12 ++-- Library/Homebrew/test/cmd/log_spec.rb | 12 +++- Library/Homebrew/test/cmd/migrate_spec.rb | 31 +++++++-- Library/Homebrew/test/cmd/missing_spec.rb | 10 ++- Library/Homebrew/test/cmd/nodenv-sync_spec.rb | 1 + Library/Homebrew/test/cmd/pin_spec.rb | 11 ++-- Library/Homebrew/test/cmd/postinstall_spec.rb | 14 ++++ Library/Homebrew/test/cmd/pyenv-sync_spec.rb | 1 + Library/Homebrew/test/cmd/rbenv-sync_spec.rb | 1 + Library/Homebrew/test/cmd/reinstall_spec.rb | 9 ++- .../Homebrew/test/cmd/sandbox-exec_spec.rb | 1 + Library/Homebrew/test/cmd/source_spec.rb | 10 ++- Library/Homebrew/test/cmd/tab_spec.rb | 19 ++++-- Library/Homebrew/test/cmd/uninstall_spec.rb | 12 ++-- Library/Homebrew/test/cmd/unpin_spec.rb | 11 ++-- .../test/cmd/update-if-needed_spec.rb | 6 ++ .../Homebrew/test/cmd/update-reset_spec.rb | 6 ++ Library/Homebrew/test/cmd/update_spec.rb | 1 + Library/Homebrew/test/cmd/uses_spec.rb | 10 +++ .../Homebrew/test/cmd/version-install_spec.rb | 19 ++++-- Library/Homebrew/test/cmd/vulns_spec.rb | 9 +++ .../Homebrew/test/cmd/which-formula_spec.rb | 1 + Library/Homebrew/test/dev-cmd/audit_spec.rb | 7 ++ .../Homebrew/test/dev-cmd/benchmark_spec.rb | 37 +++++++++++ .../test/dev-cmd/bump-cask-pr_spec.rb | 13 ++++ .../bump-compatibility-version_spec.rb | 9 +++ .../test/dev-cmd/bump-formula-pr_spec.rb | 15 +++++ .../test/dev-cmd/bump-revision_spec.rb | 9 +++ .../dev-cmd/bump-unversioned-casks_spec.rb | 17 +++++ Library/Homebrew/test/dev-cmd/bump_spec.rb | 8 +-- Library/Homebrew/test/dev-cmd/cat_spec.rb | 7 +- .../test/dev-cmd/contributions_spec.rb | 1 + .../Homebrew/test/dev-cmd/debugger_spec.rb | 6 ++ Library/Homebrew/test/dev-cmd/edit_spec.rb | 6 +- .../dev-cmd/generate-man-completions_spec.rb | 1 + .../test/dev-cmd/generate-zap_spec.rb | 18 +++++ .../test/dev-cmd/install-bundler-gems_spec.rb | 1 + Library/Homebrew/test/dev-cmd/lgtm_spec.rb | 1 + Library/Homebrew/test/dev-cmd/linkage_spec.rb | 8 ++- .../Homebrew/test/dev-cmd/livecheck_spec.rb | 10 ++- Library/Homebrew/test/dev-cmd/rubocop_spec.rb | 6 ++ Library/Homebrew/test/dev-cmd/rubydoc_spec.rb | 6 ++ Library/Homebrew/test/dev-cmd/style_spec.rb | 16 +++++ .../Homebrew/test/dev-cmd/test-bot_spec.rb | 6 ++ Library/Homebrew/test/dev-cmd/tests_spec.rb | 1 + .../Homebrew/test/dev-cmd/typecheck_spec.rb | 1 + Library/Homebrew/test/dev-cmd/unpack_spec.rb | 12 ++-- .../dev-cmd/update-perl-resources_spec.rb | 8 +++ .../dev-cmd/update-python-resources_spec.rb | 11 ++++ .../Homebrew/test/dev-cmd/update-test_spec.rb | 1 + .../Homebrew/test/dev-cmd/vendor-gems_spec.rb | 1 + Library/Homebrew/test/dev-cmd/verify_spec.rb | 8 +++ Library/Homebrew/test/manpages_spec.rb | 14 ++++ Library/Homebrew/test/spec_helper.rb | 2 + .../support/helper/integration_coverage.rb | 63 ++++++++++++++++++ .../helper/integration_coverage_spec.rb | 55 ++++++++++++++++ .../test/support/helper/integration_mocks.rb | 2 +- .../test/support/helper/simplecov_start.rb | 9 --- .../spec/shared_context/integration_test.rb | 65 +++++++++---------- .../shared_context/integration_test_spec.rb | 50 ++++++++++++++ 79 files changed, 701 insertions(+), 180 deletions(-) create mode 100644 Library/Homebrew/test/cmd/--taps_spec.rb create mode 100644 Library/Homebrew/test/cmd/update-if-needed_spec.rb create mode 100644 Library/Homebrew/test/cmd/update-reset_spec.rb create mode 100644 Library/Homebrew/test/dev-cmd/debugger_spec.rb create mode 100644 Library/Homebrew/test/dev-cmd/rubocop_spec.rb create mode 100644 Library/Homebrew/test/dev-cmd/rubydoc_spec.rb create mode 100644 Library/Homebrew/test/dev-cmd/test-bot_spec.rb create mode 100644 Library/Homebrew/test/support/helper/integration_coverage.rb create mode 100644 Library/Homebrew/test/support/helper/integration_coverage_spec.rb delete mode 100644 Library/Homebrew/test/support/helper/simplecov_start.rb create mode 100644 Library/Homebrew/test/support/helper/spec/shared_context/integration_test_spec.rb diff --git a/Library/Homebrew/.simplecov b/Library/Homebrew/.simplecov index b174477f9073f..b28c416af815c 100755 --- a/Library/Homebrew/.simplecov +++ b/Library/Homebrew/.simplecov @@ -34,52 +34,8 @@ SimpleCov.configure do .map { |p| "#{p}/**/*.rb" }.join(",") files = "{#{subdirs},*.rb}" - if (integration_test_number = ENV.fetch("HOMEBREW_INTEGRATION_TEST", nil)) - # This needs a unique name so it won't be overwritten - command_name "brew_i:#{integration_test_number}" - - # be quiet, the parent process will be in charge of output and checking coverage totals - SimpleCov.print_errors false - - SimpleCov.at_exit do - # Just save result, but don't write formatted output. - coverage_result = Coverage.result.dup - simulated_files_path = File.join( - SimpleCov.coverage_path, - ".simulated_files#{ENV.fetch("TEST_ENV_NUMBER", "")}", - ) - simulated_files = if File.exist?(simulated_files_path) - Set.new(File.readlines(simulated_files_path, chomp: true)) - else - Set.new - end - simulated_files_count = simulated_files.size - Dir.glob(files, base: SimpleCov.root).each do |file| - absolute_path = File.expand_path(file, SimpleCov.root) - next if coverage_result.key?(absolute_path) || simulated_files.include?(absolute_path) - - coverage_result[absolute_path] = SimpleCov::SimulateCoverage.call(absolute_path) - simulated_files << absolute_path - end - simplecov_result = SimpleCov::Result.new(coverage_result) - SimpleCov::ResultMerger.store_result(simplecov_result) - if simulated_files.size > simulated_files_count - FileUtils.mkdir_p(SimpleCov.coverage_path) - File.write(simulated_files_path, simulated_files.to_a.join("\n")) - end - - # If an integration test raises a `SystemExit` exception on exit, - # exit immediately using the same status code to avoid reporting - # an error when expecting a non-successful exit status. - raise if $ERROR_INFO.is_a?(SystemExit) - end - else - command_name "brew:#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}" - - # Not using this during integration tests makes the tests 4x times faster - # without changing the coverage. - cover files - end + command_name "brew:#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}" + cover files skip(/^build\.rb$/) skip(/^config\.rb$/) diff --git a/Library/Homebrew/dev-cmd/tests.rb b/Library/Homebrew/dev-cmd/tests.rb index e3e10dbc22e02..6ef148352d77b 100644 --- a/Library/Homebrew/dev-cmd/tests.rb +++ b/Library/Homebrew/dev-cmd/tests.rb @@ -282,7 +282,6 @@ def setup_environment! if args.coverage? ENV["HOMEBREW_TESTS_COVERAGE"] = "1" FileUtils.rm_f "test/coverage/.resultset.json" - FileUtils.rm_f Dir["test/coverage/.simulated_files*"] end # Override author/committer as global settings might be invalid and thus diff --git a/Library/Homebrew/test/cmd/--caskroom_spec.rb b/Library/Homebrew/test/cmd/--caskroom_spec.rb index 04d81c8da2678..496ad646c8a12 100644 --- a/Library/Homebrew/test/cmd/--caskroom_spec.rb +++ b/Library/Homebrew/test/cmd/--caskroom_spec.rb @@ -7,11 +7,24 @@ RSpec.describe Homebrew::Cmd::Caskroom do it_behaves_like "parseable arguments" - it "prints Homebrew's Caskroom", :integration_test do + it "prints Homebrew's Caskroom and a Cask's Caskroom", :cask, :integration_test do + caskfile = HOMEBREW_TEMP/"local-caffeine.rb" + caskfile.write <<~RUBY + cask "local-caffeine" do + version "1.2.3" + sha256 :no_check + url "https://brew.sh/local-caffeine.zip" + end + RUBY + expect { brew_sh "--caskroom" } .to output("#{ENV.fetch("HOMEBREW_PREFIX")}/Caskroom\n").to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew_sh "--caskroom", caskfile } + .to output("#{ENV.fetch("HOMEBREW_PREFIX")}/Caskroom/local-caffeine\n").to_stdout + .and not_to_output.to_stderr + .and be_a_success end it "prints the Caskroom for Casks" do diff --git a/Library/Homebrew/test/cmd/--cellar_spec.rb b/Library/Homebrew/test/cmd/--cellar_spec.rb index 254b68ad345ff..5e82b74c27ec9 100644 --- a/Library/Homebrew/test/cmd/--cellar_spec.rb +++ b/Library/Homebrew/test/cmd/--cellar_spec.rb @@ -7,11 +7,17 @@ RSpec.describe Homebrew::Cmd::Cellar do it_behaves_like "parseable arguments" - it "prints Homebrew's Cellar", :integration_test do + it "prints Homebrew's Cellar and a Formula's Cellar", :integration_test do + testball = setup_test_formula "testball" + expect { brew_sh "--cellar" } .to output("#{ENV.fetch("HOMEBREW_CELLAR")}\n").to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew_sh "--cellar", testball } + .to output(%r{/Cellar/testball\n}).to_stdout + .and not_to_output.to_stderr + .and be_a_success end it "prints the Cellar for a Formula" do diff --git a/Library/Homebrew/test/cmd/--env_spec.rb b/Library/Homebrew/test/cmd/--env_spec.rb index bbcfb05c10ff9..fc612dec629fe 100644 --- a/Library/Homebrew/test/cmd/--env_spec.rb +++ b/Library/Homebrew/test/cmd/--env_spec.rb @@ -9,11 +9,16 @@ describe "--shell=bash", :integration_test do it "prints the Homebrew build environment variables in Bash syntax" do + setup_test_formula "testball" path = [Superenv.bin&.parent, HOMEBREW_PREFIX].compact.join(File::PATH_SEPARATOR) expect { brew "--env", "--shell=bash" } .to output(/export CMAKE_PREFIX_PATH="#{Regexp.quote(path)}"/).to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew "--env", "--shell=bash", "testball" } + .to output(/export CMAKE_PREFIX_PATH="#{Regexp.quote(path)}"/).to_stdout + .and not_to_output.to_stderr + .and be_a_success end end end diff --git a/Library/Homebrew/test/cmd/--prefix_spec.rb b/Library/Homebrew/test/cmd/--prefix_spec.rb index 21c58a268de6a..0c2e892a24367 100644 --- a/Library/Homebrew/test/cmd/--prefix_spec.rb +++ b/Library/Homebrew/test/cmd/--prefix_spec.rb @@ -7,11 +7,17 @@ RSpec.describe Homebrew::Cmd::Prefix do it_behaves_like "parseable arguments" - it "prints Homebrew's prefix", :integration_test do + it "prints Homebrew's prefix and a Formula's prefix", :integration_test do + testball = setup_test_formula "testball" + expect { brew_sh "--prefix" } .to output("#{ENV.fetch("HOMEBREW_PREFIX")}\n").to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew_sh "--prefix", testball } + .to output(%r{/opt/testball\n}).to_stdout + .and not_to_output.to_stderr + .and be_a_success end it "prints the prefix for a Formula" do diff --git a/Library/Homebrew/test/cmd/--repository_spec.rb b/Library/Homebrew/test/cmd/--repository_spec.rb index 545a330187f82..7fd16c941f0d1 100644 --- a/Library/Homebrew/test/cmd/--repository_spec.rb +++ b/Library/Homebrew/test/cmd/--repository_spec.rb @@ -4,6 +4,8 @@ require "open3" RSpec.describe "brew --repository", type: :system do + it_behaves_like "a documented command", "--repository", shell: true + it "prints Homebrew and Tap repositories" do stdout, stderr, status = Open3.capture3( { diff --git a/Library/Homebrew/test/cmd/--taps_spec.rb b/Library/Homebrew/test/cmd/--taps_spec.rb new file mode 100644 index 0000000000000..87367b2efee61 --- /dev/null +++ b/Library/Homebrew/test/cmd/--taps_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew --taps", type: :system do + it_behaves_like "a documented command", "--taps", shell: true +end diff --git a/Library/Homebrew/test/cmd/alias_spec.rb b/Library/Homebrew/test/cmd/alias_spec.rb index d3cabf26afd53..760faa0f4d107 100644 --- a/Library/Homebrew/test/cmd/alias_spec.rb +++ b/Library/Homebrew/test/cmd/alias_spec.rb @@ -12,9 +12,9 @@ .to not_to_output.to_stdout .and not_to_output.to_stderr .and be_a_success - expect { brew "alias" } - .to output(/brew alias foo-test='bar'/).to_stdout - .and not_to_output.to_stderr - .and be_a_success + + alias_file = HOMEBREW_ALIASES/"foo_test" + expect([alias_file.read.include?("brew bar $*"), (HOMEBREW_PREFIX/"bin/brew-foo-test").realpath]) + .to eq([true, alias_file]) end end diff --git a/Library/Homebrew/test/cmd/analytics_spec.rb b/Library/Homebrew/test/cmd/analytics_spec.rb index 64f4b2b4c4121..7cf340a4eb52b 100644 --- a/Library/Homebrew/test/cmd/analytics_spec.rb +++ b/Library/Homebrew/test/cmd/analytics_spec.rb @@ -16,15 +16,17 @@ .to raise_error(Homebrew::CLI::MaxNamedArgumentsError) end - it "when HOMEBREW_NO_ANALYTICS is unset is disabled after running `brew analytics off`", :integration_test do + it "disables analytics when HOMEBREW_NO_ANALYTICS is unset", :integration_test do HOMEBREW_REPOSITORY.cd do system "git", "init" end - brew "analytics", "off" - expect { brew "analytics", "HOMEBREW_NO_ANALYTICS" => nil } - .to output(/analytics are disabled/i).to_stdout + expect { brew "analytics", "off", "HOMEBREW_NO_ANALYTICS" => nil } + .to not_to_output.to_stdout .and not_to_output.to_stderr .and be_a_success + + Homebrew::Settings.clear_cache + expect(Homebrew::Settings.read(:analyticsdisabled)).to eq("true") end end diff --git a/Library/Homebrew/test/cmd/as-console-user_spec.rb b/Library/Homebrew/test/cmd/as-console-user_spec.rb index 39a0def9e10fb..0452bfac577c1 100644 --- a/Library/Homebrew/test/cmd/as-console-user_spec.rb +++ b/Library/Homebrew/test/cmd/as-console-user_spec.rb @@ -21,6 +21,7 @@ end it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "as-console-user", shell: true def run_as_console_user_shell(script, env = {}) Bundler.with_unbundled_env do diff --git a/Library/Homebrew/test/cmd/cleanup_spec.rb b/Library/Homebrew/test/cmd/cleanup_spec.rb index 6e04c92844c50..493525d58e133 100644 --- a/Library/Homebrew/test/cmd/cleanup_spec.rb +++ b/Library/Homebrew/test/cmd/cleanup_spec.rb @@ -16,14 +16,17 @@ it_behaves_like "parseable arguments" - describe "--prune=all", :integration_test do + describe "--prune=all", :cask, :integration_test do it "removes all files in Homebrew's cache" do + setup_test_formula "testball" (HOMEBREW_CACHE/"test").write "test" expect { brew "cleanup", "--prune=all" } .to output(%r{#{Regexp.escape(HOMEBREW_CACHE)}/test}o).to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew "cleanup", "--dry-run", "testball", cask_path("local-caffeine") } + .to be_a_success end end end diff --git a/Library/Homebrew/test/cmd/command-not-found-init_spec.rb b/Library/Homebrew/test/cmd/command-not-found-init_spec.rb index 051c78f077dec..f4fafb376c00b 100644 --- a/Library/Homebrew/test/cmd/command-not-found-init_spec.rb +++ b/Library/Homebrew/test/cmd/command-not-found-init_spec.rb @@ -6,4 +6,5 @@ RSpec.describe Homebrew::Cmd::CommandNotFoundInit do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "command-not-found-init" end diff --git a/Library/Homebrew/test/cmd/completions_spec.rb b/Library/Homebrew/test/cmd/completions_spec.rb index 3b3677cd3df31..5e39195b134c1 100644 --- a/Library/Homebrew/test/cmd/completions_spec.rb +++ b/Library/Homebrew/test/cmd/completions_spec.rb @@ -16,15 +16,17 @@ .to raise_error(Homebrew::CLI::MaxNamedArgumentsError) end - it "runs the status subcommand correctly", :integration_test do + it "links completions", :integration_test do HOMEBREW_REPOSITORY.cd do system "git", "init" end - brew "completions", "link" - expect { brew "completions" } - .to output(/Completions are linked/).to_stdout + expect { brew "completions", "link" } + .to output(/Completions are now linked/).to_stdout .and not_to_output.to_stderr .and be_a_success + + Homebrew::Settings.clear_cache + expect(Homebrew::Completions.link_completions?).to be(true) end end diff --git a/Library/Homebrew/test/cmd/deps_spec.rb b/Library/Homebrew/test/cmd/deps_spec.rb index 532adbdcc93f6..25407eadd6898 100644 --- a/Library/Homebrew/test/cmd/deps_spec.rb +++ b/Library/Homebrew/test/cmd/deps_spec.rb @@ -36,14 +36,19 @@ it_behaves_like "parseable arguments" - it "outputs all of a Formula's dependencies and their dependencies on separate lines" do + it "outputs Formula and Cask dependencies on separate lines", :cask do setup_test_formula "installed" + setup_test_formula "unar" expect do brew "deps", "baz", "--include-test", "--missing", "--skip-recommended", "HOMEBREW_REQUIRE_TAP_TRUST" => "1" end .to be_a_success .and output("bar\nfoo\ntest\n").to_stdout .and output(/not the actual runtime dependencies/).to_stderr + expect { brew "deps", "--cask", "--direct", cask_path("with-depends-on-everything") } + .to output("unar\n").to_stdout + .and output(/not the actual runtime dependencies/).to_stderr + .and be_a_success end context "with --tree" do diff --git a/Library/Homebrew/test/cmd/desc_spec.rb b/Library/Homebrew/test/cmd/desc_spec.rb index 6196ea5513c30..71cbc2cf8a77d 100644 --- a/Library/Homebrew/test/cmd/desc_spec.rb +++ b/Library/Homebrew/test/cmd/desc_spec.rb @@ -7,11 +7,11 @@ RSpec.describe Homebrew::Cmd::Desc do it_behaves_like "parseable arguments" - it "shows a given Formula's description", :integration_test do + it "shows a given Formula and Cask description", :cask, :integration_test do setup_test_formula "testball" - expect { brew "desc", "testball" } - .to output("testball: Some test\n").to_stdout + expect { brew "desc", "testball", "local-transmission" } + .to output("local-transmission: (Transmission) BitTorrent client\ntestball: Some test\n").to_stdout .and not_to_output.to_stderr .and be_a_success end diff --git a/Library/Homebrew/test/cmd/gist-logs_spec.rb b/Library/Homebrew/test/cmd/gist-logs_spec.rb index a6e62273e6195..d81be9fb3b8fd 100644 --- a/Library/Homebrew/test/cmd/gist-logs_spec.rb +++ b/Library/Homebrew/test/cmd/gist-logs_spec.rb @@ -7,6 +7,14 @@ RSpec.describe Homebrew::Cmd::GistLogs do it_behaves_like "parseable arguments" + it "loads a Formula's build logs", :integration_test do + setup_test_formula "testball" + + expect { brew "gist-logs", "testball", "HOMEBREW_TEST_GENERIC_OS" => "1" } + .to output(/No logs\./).to_stderr + .and be_a_failure + end + describe ".truncate_text_to_approximate_size" do let(:glue) { "\n[...snip...]\n" } # hard-coded copy from truncate_text_to_approximate_size diff --git a/Library/Homebrew/test/cmd/home_spec.rb b/Library/Homebrew/test/cmd/home_spec.rb index 1c7cee0963b6c..1a0aae253b6d3 100644 --- a/Library/Homebrew/test/cmd/home_spec.rb +++ b/Library/Homebrew/test/cmd/home_spec.rb @@ -26,7 +26,15 @@ it_behaves_like "parseable arguments" - it "opens the project page when no formula or cask is specified", :integration_test do + it "opens the homepages for a given Formula and Cask", :cask, :integration_test do + setup_test_formula "testballhome", <<~RUBY + homepage "#{testballhome_homepage}" + RUBY + + expect { brew "home", "testballhome", local_caffeine_path, "HOMEBREW_BROWSER" => "echo" } + .to output(/#{Regexp.escape(testballhome_homepage)}.*#{Regexp.escape(local_caffeine_homepage)}/m).to_stdout + .and output(/Treating #{Regexp.escape(local_caffeine_path)} as a cask/).to_stderr + .and be_a_success expect { brew "home", "HOMEBREW_BROWSER" => "echo" } .to output("https://brew.sh\n").to_stdout .and not_to_output.to_stderr diff --git a/Library/Homebrew/test/cmd/info_spec.rb b/Library/Homebrew/test/cmd/info_spec.rb index 3edfa1587149c..9e81cfec473f5 100644 --- a/Library/Homebrew/test/cmd/info_spec.rb +++ b/Library/Homebrew/test/cmd/info_spec.rb @@ -55,10 +55,10 @@ def installed_info_cask .and not_to_output.to_stderr end - it "prints as json with the --json=v2 flag", :integration_test do + it "prints Formula and Cask JSON with the --json=v2 flag", :cask, :integration_test do setup_test_formula "testball" - expect { brew "info", "testball", "--json=v2" } + expect { brew "info", "testball", "local-caffeine", "--json=v2" } .to output(a_json_string).to_stdout .and not_to_output.to_stderr .and be_a_success diff --git a/Library/Homebrew/test/cmd/install_spec.rb b/Library/Homebrew/test/cmd/install_spec.rb index 32586a3a61c07..69d382a77067c 100644 --- a/Library/Homebrew/test/cmd/install_spec.rb +++ b/Library/Homebrew/test/cmd/install_spec.rb @@ -560,7 +560,7 @@ end context "when installing Formulae" do - it "builds from source and pours a keg-only bottle", :integration_test do + it "installs Formulae and a Cask", :cask, :integration_test do source_formula_name = "sourceball" source_formula_prefix = HOMEBREW_CELLAR/source_formula_name/"0.1" bottle_formula_name = "testball_bottle" @@ -577,11 +577,13 @@ def install setup_test_formula bottle_formula_name, <<~RUBY keg_only "test reason" RUBY + appdir = mktmpdir with_env(HOMEBREW_NO_INSTALL_FROM_API: "1") do expect do - brew "install", "--yes", source_formula_name, bottle_formula_name, - "HOMEBREW_NO_INSTALL_FROM_API" => "1" + brew "install", "--yes", "--no-ask", "--appdir=#{appdir}", source_formula_name, bottle_formula_name, + cask_path("local-caffeine"), + "HOMEBREW_NO_INSTALL_FROM_API" => "1", "HOMEBREW_TEST_GENERIC_OS" => "1" end .to output(/#{Regexp.escape(source_formula_prefix)}.*#{Regexp.escape(bottle_formula_prefix)}/m).to_stdout .and output(/✔︎.*/m).to_stderr @@ -591,6 +593,7 @@ def install expect(bottle_formula_prefix/"foo/test").not_to be_a_file expect(bottle_formula_prefix/"bin/helloworld").to be_a_file expect(HOMEBREW_PREFIX/"bin/helloworld").not_to be_a_file + expect(appdir/"Caffeine.app").to be_a_directory end end @@ -625,7 +628,8 @@ def install expect do brew "install", "-y", formula_name, "--HEAD", "HOMEBREW_DOWNLOAD_CONCURRENCY" => "1", - "HOMEBREW_NO_INSTALL_FROM_API" => "1" + "HOMEBREW_NO_INSTALL_FROM_API" => "1", + "HOMEBREW_TEST_GENERIC_OS" => "1" end .to output(/#{Regexp.escape(testball1_prefix)}/o).to_stdout .and output(/Cloning into/).to_stderr diff --git a/Library/Homebrew/test/cmd/log_spec.rb b/Library/Homebrew/test/cmd/log_spec.rb index a5d1d32462c0e..3196678eb0484 100644 --- a/Library/Homebrew/test/cmd/log_spec.rb +++ b/Library/Homebrew/test/cmd/log_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Homebrew::Cmd::Log do it_behaves_like "parseable arguments" - it "shows the Git log for a given Formula", :integration_test do + it "shows the Git log for a given Formula and Cask", :cask, :integration_test do setup_test_formula "testball" core_tap = CoreTap.instance @@ -28,5 +28,15 @@ .and be_a_success expect(shallow_tap.path/".git/shallow").to exist, "A shallow clone should have been created." + + CoreCaskTap.instance.path.cd do + system "git", "init" + system "git", "add", "--all" + system "git", "commit", "-m", "This is a test commit for local-caffeine" + end + expect { brew "log", "--cask", "local-caffeine" } + .to output(/This is a test commit for local-caffeine/).to_stdout + .and not_to_output.to_stderr + .and be_a_success end end diff --git a/Library/Homebrew/test/cmd/migrate_spec.rb b/Library/Homebrew/test/cmd/migrate_spec.rb index a80fe4473efb1..09031daed79b0 100644 --- a/Library/Homebrew/test/cmd/migrate_spec.rb +++ b/Library/Homebrew/test/cmd/migrate_spec.rb @@ -7,14 +7,37 @@ RSpec.describe Homebrew::Cmd::Migrate do it_behaves_like "parseable arguments" - it "migrates a renamed Formula", :integration_test, :no_api do - setup_test_formula "testball1" + it "migrates a renamed Formula and Cask", :cask, :integration_test, :no_api do + old_formula_path = setup_test_formula "testball1", tab_attributes: { installed_on_request: true } setup_test_formula "testball2" - install_and_rename_coretap_formula "testball1", "testball2" + Keg.new(::Formula["testball1"].prefix).link + old_formula_path.unlink + (CoreTap.instance.path/"formula_renames.json").write( + JSON.pretty_generate("testball1" => "testball2"), + ) + CoreTap.instance.clear_cache - expect { brew "migrate", "testball1" } + expect { brew "migrate", "testball1", "HOMEBREW_TEST_GENERIC_OS" => "1" } .to output(/Migrating formula testball1 to testball2/).to_stdout .and not_to_output.to_stderr .and be_a_success + + cask = Cask::CaskLoader.load("local-caffeine") + InstallHelper.stub_cask_installation(cask) + sourcefile_path = cask.sourcefile_path + raise "Cask sourcefile path is unavailable" if sourcefile_path.nil? + + sourcefile_path.unlink + (CoreCaskTap.instance.path/"cask_renames.json").write( + JSON.pretty_generate("local-caffeine" => "local-transmission-zip"), + ) + CoreCaskTap.instance.clear_cache + + expect { brew "migrate", "--cask", "--dry-run", cask.token } + .to output(/Would migrate cask local-caffeine to local-transmission-zip/).to_stdout + .and not_to_output.to_stderr + .and be_a_success + expect(HOMEBREW_CELLAR/"testball1").to be_a_symlink + expect((HOMEBREW_CELLAR/"testball1").realpath).to eq(HOMEBREW_CELLAR/"testball2") end end diff --git a/Library/Homebrew/test/cmd/missing_spec.rb b/Library/Homebrew/test/cmd/missing_spec.rb index e1d0f2577bcf8..43d19d179a424 100644 --- a/Library/Homebrew/test/cmd/missing_spec.rb +++ b/Library/Homebrew/test/cmd/missing_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Homebrew::Cmd::Missing do it_behaves_like "parseable arguments" - it "prints missing dependencies", :integration_test, :no_api do + it "prints missing Formula and Cask dependencies", :cask, :integration_test, :no_api do setup_test_formula "foo" setup_test_formula "bar" @@ -23,6 +23,14 @@ .to output("foo\n").to_stdout .and not_to_output.to_stderr .and be_a_failure + + setup_test_formula "unar" + cask = Cask::CaskLoader.load("with-depends-on-everything") + InstallHelper.stub_cask_installation(cask) + expect { brew "missing", cask.token } + .to output(/local-caffeine.*unar/).to_stdout + .and not_to_output.to_stderr + .and be_a_failure end it "prints missing cask dependencies", :cask, :no_api do diff --git a/Library/Homebrew/test/cmd/nodenv-sync_spec.rb b/Library/Homebrew/test/cmd/nodenv-sync_spec.rb index aa85a6de7460e..cbbabbce6cf65 100644 --- a/Library/Homebrew/test/cmd/nodenv-sync_spec.rb +++ b/Library/Homebrew/test/cmd/nodenv-sync_spec.rb @@ -6,4 +6,5 @@ RSpec.describe Homebrew::Cmd::NodenvSync do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "nodenv-sync" end diff --git a/Library/Homebrew/test/cmd/pin_spec.rb b/Library/Homebrew/test/cmd/pin_spec.rb index f8d39ee77df71..43ed54a1a9dc2 100644 --- a/Library/Homebrew/test/cmd/pin_spec.rb +++ b/Library/Homebrew/test/cmd/pin_spec.rb @@ -7,19 +7,16 @@ RSpec.describe Homebrew::Cmd::Pin do it_behaves_like "parseable arguments" - it "pins a Formula's version", :integration_test do + it "pins Formula and Cask versions", :cask, :integration_test do setup_test_formula "testball", tab_attributes: { installed_on_request: true } - - expect { brew "pin", "testball" }.to be_a_success - end - - it "pins a Cask's version", :cask do cask = Cask::CaskLoader.load("local-caffeine") InstallHelper.stub_cask_installation(cask) - expect { described_class.new(["--cask", "local-caffeine"]).run } + expect { brew "pin", "testball", "local-caffeine" } .to not_to_output.to_stderr + .and be_a_success + expect(Formula["testball"]).to be_pinned expect(cask).to be_pinned expect(cask.pinned_version).to eq("1.2.3") cask.unpin diff --git a/Library/Homebrew/test/cmd/postinstall_spec.rb b/Library/Homebrew/test/cmd/postinstall_spec.rb index dd70f9dffbb55..7c964f559362a 100644 --- a/Library/Homebrew/test/cmd/postinstall_spec.rb +++ b/Library/Homebrew/test/cmd/postinstall_spec.rb @@ -7,6 +7,20 @@ RSpec.describe Homebrew::Cmd::Postinstall do it_behaves_like "parseable arguments" + it "runs a Formula's post-install steps", :integration_test do + setup_test_formula "testball", <<~RUBY, tab_attributes: { installed_on_request: true } + def post_install + (prefix/"postinstall-ran").write "yes" + end + RUBY + + expect { brew "postinstall", "testball", "HOMEBREW_TEST_GENERIC_OS" => "1" } + .to output(/Postinstalling testball/).to_stdout + .and output(/Sandbox unavailable: running post-install without sandboxing!/).to_stderr + .and be_a_success + expect(Formula["testball"].prefix/"postinstall-ran").to be_a_file + end + it "runs post-install steps through `FormulaInstaller`" do cmd = described_class.new(["foo"]) formula = instance_double(Formula, install_etc_var: nil, post_install_steps_defined?: true, diff --git a/Library/Homebrew/test/cmd/pyenv-sync_spec.rb b/Library/Homebrew/test/cmd/pyenv-sync_spec.rb index c5587e7dfa047..63b54cd993889 100644 --- a/Library/Homebrew/test/cmd/pyenv-sync_spec.rb +++ b/Library/Homebrew/test/cmd/pyenv-sync_spec.rb @@ -6,4 +6,5 @@ RSpec.describe Homebrew::Cmd::PyenvSync do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "pyenv-sync" end diff --git a/Library/Homebrew/test/cmd/rbenv-sync_spec.rb b/Library/Homebrew/test/cmd/rbenv-sync_spec.rb index 2ecb201ca1ab2..bbd04f3e9a1eb 100644 --- a/Library/Homebrew/test/cmd/rbenv-sync_spec.rb +++ b/Library/Homebrew/test/cmd/rbenv-sync_spec.rb @@ -6,4 +6,5 @@ RSpec.describe Homebrew::Cmd::RbenvSync do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "rbenv-sync" end diff --git a/Library/Homebrew/test/cmd/reinstall_spec.rb b/Library/Homebrew/test/cmd/reinstall_spec.rb index b8fa2d827285b..10ab736f06896 100644 --- a/Library/Homebrew/test/cmd/reinstall_spec.rb +++ b/Library/Homebrew/test/cmd/reinstall_spec.rb @@ -227,7 +227,7 @@ expect { cmd.run }.to output(/Error: one: gzip decompression failed/).to_stderr end - it "reinstalls a Formula", :integration_test do + it "reinstalls a Formula and Cask", :cask, :integration_test do formula_name = "testball_bottle" formula_prefix = HOMEBREW_CELLAR/formula_name/"0.1" formula_bin = formula_prefix/"bin" @@ -236,8 +236,13 @@ Keg.new(formula_prefix).link expect(formula_bin).not_to exist + cask = Cask::CaskLoader.load(cask_path("local-caffeine")) + InstallHelper.stub_cask_installation(cask) - expect { brew "reinstall", formula_name } + expect do + brew "reinstall", "--no-ask", "--appdir=#{Cask::Config::DEFAULT_DIRS_PATHNAMES[:appdir]}", formula_name, + cask_path("local-caffeine"), "HOMEBREW_TEST_GENERIC_OS" => "1" + end .to output(/Reinstalling #{formula_name}/).to_stdout .and output(/✔︎.*/m).to_stderr .and be_a_success diff --git a/Library/Homebrew/test/cmd/sandbox-exec_spec.rb b/Library/Homebrew/test/cmd/sandbox-exec_spec.rb index 3a9141157358f..dadf7e61ca383 100644 --- a/Library/Homebrew/test/cmd/sandbox-exec_spec.rb +++ b/Library/Homebrew/test/cmd/sandbox-exec_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Homebrew::Cmd::SandboxExec do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "sandbox-exec" it "runs the command in the requested sandbox" do expect(Sandbox).to receive(:run_command) diff --git a/Library/Homebrew/test/cmd/source_spec.rb b/Library/Homebrew/test/cmd/source_spec.rb index f3cc4d9fa8cc5..7d01acd2ad820 100644 --- a/Library/Homebrew/test/cmd/source_spec.rb +++ b/Library/Homebrew/test/cmd/source_spec.rb @@ -7,11 +7,19 @@ RSpec.describe Homebrew::Cmd::Source do it_behaves_like "parseable arguments" - it "opens the Homebrew repo when no formula is specified", :integration_test do + it "opens Homebrew and Formula source repositories", :integration_test do + setup_test_formula "testball", <<~RUBY + url "https://github.com/Homebrew/testball/archive/refs/tags/v0.1.tar.gz" + RUBY + expect { brew "source", "HOMEBREW_BROWSER" => "echo" } .to output(%r{https://github\.com/Homebrew/brew}).to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew "source", "testball", "HOMEBREW_BROWSER" => "echo" } + .to output(%r{Opening repository for testball.*https://github\.com/Homebrew/testball}m).to_stdout + .and not_to_output.to_stderr + .and be_a_success end describe "#github_repo_url" do diff --git a/Library/Homebrew/test/cmd/tab_spec.rb b/Library/Homebrew/test/cmd/tab_spec.rb index 31936b1b6948b..868febedf2a43 100644 --- a/Library/Homebrew/test/cmd/tab_spec.rb +++ b/Library/Homebrew/test/cmd/tab_spec.rb @@ -20,19 +20,28 @@ def cask_installed_on_request?(cask) it_behaves_like "parseable arguments" - it "marks a formula as installed on request", :integration_test do + it "marks a Formula and Cask as installed on request", :cask, :integration_test do setup_test_formula "foo", tab_attributes: { "installed_on_request" => false } foo = Formula["foo"] - expect { brew "tab", "--installed-on-request", "foo" } - .to be_a_success - .and output(/foo is now marked as installed on request/).to_stdout + cask = Cask::CaskLoader.load(cask_path("local-caffeine")) + InstallHelper.install_with_caskfile(cask) + tab_path = cask.metadata_main_container_path/AbstractTab::FILENAME + + expect(tab_path).not_to exist + + expect { brew "tab", "--installed-on-request", "foo", cask.token } + .to output(/foo is now marked as installed on request.*local-caffeine is now marked as installed on request/m) + .to_stdout .and not_to_output.to_stderr + .and be_a_success expect(installed_on_request?(foo)).to be true + expect(tab_path).to exist + expect(cask_installed_on_request?(cask)).to be true end - it "marks or unmarks a cask as installed on request with a missing tab", :cask do + it "marks or unmarks a Cask as installed on request with a missing tab", :cask do cask = Cask::CaskLoader.load(cask_path("local-caffeine")) InstallHelper.install_with_caskfile(cask) tab_path = cask.metadata_main_container_path/AbstractTab::FILENAME diff --git a/Library/Homebrew/test/cmd/uninstall_spec.rb b/Library/Homebrew/test/cmd/uninstall_spec.rb index 89760c7057552..3787393f411b5 100644 --- a/Library/Homebrew/test/cmd/uninstall_spec.rb +++ b/Library/Homebrew/test/cmd/uninstall_spec.rb @@ -13,7 +13,9 @@ cask_file.dirname.mkpath FileUtils.cp cask_path("local-caffeine"), cask_file tap.clear_cache - appdir = mktmpdir + cask = Cask::CaskLoader.load(cask_file) + InstallHelper.stub_cask_installation(cask) + appdir = Pathname(cask.config.appdir) setup_test_formula "testball", tab_attributes: { installed_on_request: true } @@ -28,14 +30,10 @@ ENV["HOMEBREW_FORBID_PACKAGES_FROM_PATHS"] = "1" ENV["HOMEBREW_REQUIRE_TAP_TRUST"] = "1" ENV["HOMEBREW_NO_INSTALL_FROM_API"] = nil - brew_env = { "HOMEBREW_SORBET_RUNTIME" => nil, "HOMEBREW_SORBET_RECURSIVE" => nil } expect do - brew "install", "--cask", "--no-ask", "--appdir=#{appdir}", "./Casks/l/local-caffeine.rb", brew_env + brew "uninstall", "--cask", "./Casks/l/local-caffeine.rb", + "HOMEBREW_CASK_OPTS" => "--appdir=#{appdir}" end - .to output(/local-caffeine was successfully installed/).to_stdout - .and be_a_success - - expect { brew "uninstall", "--cask", "./Casks/l/local-caffeine.rb", brew_env } .to output(/Uninstalling Cask local-caffeine/).to_stdout .and not_to_output.to_stderr .and be_a_success diff --git a/Library/Homebrew/test/cmd/unpin_spec.rb b/Library/Homebrew/test/cmd/unpin_spec.rb index 77d81134d0d00..7d7da109a8066 100644 --- a/Library/Homebrew/test/cmd/unpin_spec.rb +++ b/Library/Homebrew/test/cmd/unpin_spec.rb @@ -7,21 +7,18 @@ RSpec.describe Homebrew::Cmd::Unpin do it_behaves_like "parseable arguments" - it "unpins a Formula's version", :integration_test do + it "unpins Formula and Cask versions", :cask, :integration_test do setup_test_formula "testball", tab_attributes: { installed_on_request: true } Formula["testball"].pin - - expect { brew "unpin", "testball" }.to be_a_success - end - - it "unpins a Cask's version", :cask do cask = Cask::CaskLoader.load("local-caffeine") InstallHelper.stub_cask_installation(cask) cask.pin - expect { described_class.new(["--cask", "local-caffeine"]).run } + expect { brew "unpin", "testball", "local-caffeine" } .to not_to_output.to_stderr + .and be_a_success + expect(Formula["testball"]).not_to be_pinned expect(cask).not_to be_pinned end diff --git a/Library/Homebrew/test/cmd/update-if-needed_spec.rb b/Library/Homebrew/test/cmd/update-if-needed_spec.rb new file mode 100644 index 0000000000000..0db519a3562b6 --- /dev/null +++ b/Library/Homebrew/test/cmd/update-if-needed_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew update-if-needed", type: :system do + it_behaves_like "a documented command", "update-if-needed", shell: true +end diff --git a/Library/Homebrew/test/cmd/update-reset_spec.rb b/Library/Homebrew/test/cmd/update-reset_spec.rb new file mode 100644 index 0000000000000..d5105b67d2a46 --- /dev/null +++ b/Library/Homebrew/test/cmd/update-reset_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew update-reset", type: :system do + it_behaves_like "a documented command", "update-reset", shell: true +end diff --git a/Library/Homebrew/test/cmd/update_spec.rb b/Library/Homebrew/test/cmd/update_spec.rb index c257c8a4653c8..4c594944b8c30 100644 --- a/Library/Homebrew/test/cmd/update_spec.rb +++ b/Library/Homebrew/test/cmd/update_spec.rb @@ -19,6 +19,7 @@ end it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "update", shell: true def run_update_shell(script, env) Bundler.with_unbundled_env do diff --git a/Library/Homebrew/test/cmd/uses_spec.rb b/Library/Homebrew/test/cmd/uses_spec.rb index 69e75349dccca..ab2a183002a7d 100644 --- a/Library/Homebrew/test/cmd/uses_spec.rb +++ b/Library/Homebrew/test/cmd/uses_spec.rb @@ -11,6 +11,16 @@ it_behaves_like "parseable arguments" + it "finds Formulae that use a given Formula", :integration_test, :no_api do + setup_test_formula "foo" + setup_test_formula "bar" + + expect { brew "uses", "foo", "HOMEBREW_REQUIRE_TAP_TRUST" => "1" } + .to output("bar\n").to_stdout + .and not_to_output.to_stderr + .and be_a_success + end + it "uses tap trust configuration to evaluate all formulae" do used_formula = instance_double(Formula, full_name: "foo") cmd = described_class.new(["--formula", "foo"]) diff --git a/Library/Homebrew/test/cmd/version-install_spec.rb b/Library/Homebrew/test/cmd/version-install_spec.rb index 80aa27dc6b138..c35a83102b992 100644 --- a/Library/Homebrew/test/cmd/version-install_spec.rb +++ b/Library/Homebrew/test/cmd/version-install_spec.rb @@ -17,14 +17,25 @@ let(:formula) { "foo" } before do - allow(Tap).to receive(:installed).and_return(installed_taps) - allow(Formula).to receive(:installed_formula_names).and_return(installed_formula_names) - allow(Homebrew::EnvConfig).to receive(:no_github_api?).and_return(true) - allow(Formulary).to receive(:factory) { |ref, **opts| formulary_factory.call(ref, **opts) } + unless RSpec.current_example&.metadata&.key?(:integration_test) + allow(Tap).to receive(:installed).and_return(installed_taps) + allow(Formula).to receive(:installed_formula_names).and_return(installed_formula_names) + allow(Homebrew::EnvConfig).to receive(:no_github_api?).and_return(true) + allow(Formulary).to receive(:factory) { |ref, **opts| formulary_factory.call(ref, **opts) } + end end it_behaves_like "parseable arguments" + it "recognises an installed versioned Formula", :integration_test do + setup_test_formula "testball@0.1", tab_attributes: { installed_on_request: true } + + expect { brew "version-install", "testball", "0.1" } + .to output(/testball@0\.1 is already installed/).to_stdout + .and not_to_output.to_stderr + .and be_a_success + end + context "when the versioned formula is already installed" do let(:installed_formula_names) { [versioned_name] } diff --git a/Library/Homebrew/test/cmd/vulns_spec.rb b/Library/Homebrew/test/cmd/vulns_spec.rb index cd77095d366d7..a6575ed1d07f8 100644 --- a/Library/Homebrew/test/cmd/vulns_spec.rb +++ b/Library/Homebrew/test/cmd/vulns_spec.rb @@ -8,6 +8,15 @@ RSpec.describe Homebrew::Cmd::Vulns do it_behaves_like "parseable arguments" + it "scans a Formula", :integration_test do + setup_test_formula "testball" + + expect { brew "vulns", "--json", "testball" } + .to output(/"skipped_formulae"/).to_stdout + .and not_to_output.to_stderr + .and be_a_success + end + describe "#formulae" do let(:installed) { instance_double(Formula, full_name: "curl", recursive_dependencies: []) } diff --git a/Library/Homebrew/test/cmd/which-formula_spec.rb b/Library/Homebrew/test/cmd/which-formula_spec.rb index efc58d8e2084e..ceb12404a3368 100644 --- a/Library/Homebrew/test/cmd/which-formula_spec.rb +++ b/Library/Homebrew/test/cmd/which-formula_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Homebrew::Cmd::WhichFormula do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "which-formula", shell: true describe "which_formula" do let(:shell_cellar) { HOMEBREW_CELLAR } diff --git a/Library/Homebrew/test/dev-cmd/audit_spec.rb b/Library/Homebrew/test/dev-cmd/audit_spec.rb index 3cee8d235da68..70b33770b44e6 100644 --- a/Library/Homebrew/test/dev-cmd/audit_spec.rb +++ b/Library/Homebrew/test/dev-cmd/audit_spec.rb @@ -7,6 +7,13 @@ RSpec.describe Homebrew::DevCmd::Audit do it_behaves_like "parseable arguments" + it "audits a Formula and Cask", :cask, :integration_test do + setup_test_formula "testball" + + expect { brew "audit", "--skip-style", "--only=description", "testball", "local-caffeine" } + .to be_a_success + end + describe "#run" do subject(:audit) { described_class.new(["--tap=homebrew/test"]) } diff --git a/Library/Homebrew/test/dev-cmd/benchmark_spec.rb b/Library/Homebrew/test/dev-cmd/benchmark_spec.rb index 50f3d74c40967..843ebd812136a 100644 --- a/Library/Homebrew/test/dev-cmd/benchmark_spec.rb +++ b/Library/Homebrew/test/dev-cmd/benchmark_spec.rb @@ -7,6 +7,43 @@ RSpec.describe Homebrew::DevCmd::Benchmark do it_behaves_like "parseable arguments" + it "benchmarks a Formula", :integration_test do + formula_file = setup_test_formula "testball" + (HOMEBREW_PREFIX/"bin/brew").write <<~SH + #!/bin/sh + if [ "$1" = "--cache" ]; then + printf '/tmp/testball-cache\n' + fi + SH + (HOMEBREW_PREFIX/"bin/brew").chmod(0755) + bin = mktmpdir + (bin/"hyperfine").write <<~SH + #!/bin/sh + for argument in "$@"; do + if [ "${previous:-}" = "--export-json" ]; then + export_json=$argument + fi + previous=$argument + done + phase_output=$(printf '%s\n' "$export_json" | sed 's/hyperfine-/phases-/') + printf '{"results":[{"mean":0.1}]}' > "$export_json" + printf '{"events":[]}' > "$phase_output" + SH + (bin/"hyperfine").chmod(0755) + + mktmpdir do |directory| + directory.cd do + expect do + brew "benchmark", "--runs=1", formula_file, + "PATH" => "#{bin}:#{ENV.fetch("PATH")}" + end + .to output(/Benchmark results.*Full results written/m).to_stdout + .and be_a_success + end + expect(directory/"benchmark/results.json").to be_a_file + end + end + it "rotates workloads and records hyperfine and phase timings" do directory = mktmpdir hyperfine = mktmpdir/"hyperfine" diff --git a/Library/Homebrew/test/dev-cmd/bump-cask-pr_spec.rb b/Library/Homebrew/test/dev-cmd/bump-cask-pr_spec.rb index 4f4e122d9faec..1f29f4ab851ed 100644 --- a/Library/Homebrew/test/dev-cmd/bump-cask-pr_spec.rb +++ b/Library/Homebrew/test/dev-cmd/bump-cask-pr_spec.rb @@ -75,6 +75,19 @@ it_behaves_like "parseable arguments" + it "updates a Cask without creating a pull request", :cask, :integration_test do + CoreCaskTap.instance.path.cd do + system "git", "init" + system "git", "remote", "add", "origin", "https://github.com/Homebrew/homebrew-cask" + end + + expect do + brew "bump-cask-pr", "--write-only", "--no-audit", "--no-style", + "--version=1.2.4", "--sha256=:no_check", "local-caffeine" + end.to be_a_success + expect(Cask::CaskLoader.load("local-caffeine").version.to_s).to eq("1.2.4") + end + describe "::generate_system_options" do # We simulate a macOS version older than the newest, as the method will use # the host macOS version instead of the default (the newest macOS version). diff --git a/Library/Homebrew/test/dev-cmd/bump-compatibility-version_spec.rb b/Library/Homebrew/test/dev-cmd/bump-compatibility-version_spec.rb index e29ac1a14695f..378a85eb1c53b 100644 --- a/Library/Homebrew/test/dev-cmd/bump-compatibility-version_spec.rb +++ b/Library/Homebrew/test/dev-cmd/bump-compatibility-version_spec.rb @@ -7,6 +7,15 @@ RSpec.describe Homebrew::DevCmd::BumpCompatibilityVersion do it_behaves_like "parseable arguments" + it "previews a Formula compatibility version bump", :integration_test do + setup_test_formula "testball" + + expect { brew "bump-compatibility-version", "--dry-run", "testball" } + .to output(/add "compatibility_version 1"/).to_stdout + .and not_to_output.to_stderr + .and be_a_success + end + describe "#run" do before do allow(Homebrew).to receive(:install_bundler_gems!) diff --git a/Library/Homebrew/test/dev-cmd/bump-formula-pr_spec.rb b/Library/Homebrew/test/dev-cmd/bump-formula-pr_spec.rb index 56bdb6d62e04a..273eb8b2720d2 100644 --- a/Library/Homebrew/test/dev-cmd/bump-formula-pr_spec.rb +++ b/Library/Homebrew/test/dev-cmd/bump-formula-pr_spec.rb @@ -17,6 +17,21 @@ it_behaves_like "parseable arguments" + it "updates a Formula without creating a pull request", :integration_test do + formula_path = setup_test_formula "testball" + CoreTap.instance.path.cd do + system "git", "init" + system "git", "remote", "add", "origin", "https://github.com/Homebrew/homebrew-core" + end + tarball = TEST_FIXTURE_DIR/"tarballs/testball2-0.1.tbz" + + expect do + brew "bump-formula-pr", "--write-only", "--no-audit", "--version=0.2", + "--url=file://#{tarball}", "--sha256=#{tarball.sha256}", "testball" + end.to be_a_success + expect(formula_path.read).to include("version \"0.2\"") + end + describe "#run" do it "adds updated mirrors as string literals" do formula_path = CoreTap.instance.new_formula_path("couchdb") diff --git a/Library/Homebrew/test/dev-cmd/bump-revision_spec.rb b/Library/Homebrew/test/dev-cmd/bump-revision_spec.rb index cf951a5f6541f..e116afdac8662 100644 --- a/Library/Homebrew/test/dev-cmd/bump-revision_spec.rb +++ b/Library/Homebrew/test/dev-cmd/bump-revision_spec.rb @@ -6,4 +6,13 @@ RSpec.describe Homebrew::DevCmd::BumpRevision do it_behaves_like "parseable arguments" + + it "previews a Formula revision bump", :integration_test do + setup_test_formula "testball" + + expect { brew "bump-revision", "--dry-run", "testball" } + .to output(/add "revision 1"/).to_stdout + .and not_to_output.to_stderr + .and be_a_success + end end diff --git a/Library/Homebrew/test/dev-cmd/bump-unversioned-casks_spec.rb b/Library/Homebrew/test/dev-cmd/bump-unversioned-casks_spec.rb index 9caf6381fb64d..2121affe99a5f 100644 --- a/Library/Homebrew/test/dev-cmd/bump-unversioned-casks_spec.rb +++ b/Library/Homebrew/test/dev-cmd/bump-unversioned-casks_spec.rb @@ -6,4 +6,21 @@ RSpec.describe Homebrew::DevCmd::BumpUnversionedCasks do it_behaves_like "parseable arguments" + + it "skips an unsuitable Cask", :cask, :integration_test do + caskfile = CoreCaskTap.instance.cask_dir/"versioned-test.rb" + caskfile.write <<~RUBY + cask "versioned-test" do + version "1.0" + sha256 :no_check + url "https://brew.sh/versioned-test-1.0.zip" + end + RUBY + CoreCaskTap.instance.clear_cache + + expect { brew "bump-unversioned-casks", "--dry-run", caskfile } + .to output(/Unversioned Casks: 1.*Checking versioned-test/m).to_stdout + .and output(/Skipping, not a single-app or PKG cask/).to_stderr + .and be_a_success + end end diff --git a/Library/Homebrew/test/dev-cmd/bump_spec.rb b/Library/Homebrew/test/dev-cmd/bump_spec.rb index 9ab6ba3ca0478..736dc901fa049 100644 --- a/Library/Homebrew/test/dev-cmd/bump_spec.rb +++ b/Library/Homebrew/test/dev-cmd/bump_spec.rb @@ -41,8 +41,8 @@ it_behaves_like "parseable arguments" - describe "formula", :integration_test, :needs_homebrew_curl, :needs_network do - it "returns no data and prints a message for HEAD-only formulae" do + describe "formula and cask", :cask, :integration_test do + it "prints messages for HEAD-only Formulae and latest Casks" do content = <<~RUBY desc "HEAD-only test formula" homepage "https://brew.sh" @@ -50,8 +50,8 @@ RUBY setup_test_formula("headonly", content) - expect { brew "bump", "headonly" } - .to output(/Formula is HEAD-only./).to_stdout + expect { brew "bump", "--no-pull-requests", "headonly", "version-latest" } + .to output(/Formula is HEAD-only.*Cask uses `version :latest`/m).to_stdout .and not_to_output.to_stderr .and be_a_success end diff --git a/Library/Homebrew/test/dev-cmd/cat_spec.rb b/Library/Homebrew/test/dev-cmd/cat_spec.rb index 2d7ed7d3ed39a..e05c5337dce27 100644 --- a/Library/Homebrew/test/dev-cmd/cat_spec.rb +++ b/Library/Homebrew/test/dev-cmd/cat_spec.rb @@ -33,12 +33,11 @@ class Testball < Formula cat.run end - it "prints the content of a given Formula", :integration_test do + it "prints the content of a given Formula and Cask", :cask, :integration_test do formula_file = setup_test_formula "testball" - content = formula_file.read - expect { brew "cat", "testball" } - .to output(content).to_stdout + expect { brew "cat", "testball", "local-caffeine" } + .to output(/#{Regexp.escape(formula_file.read)}.*#{Regexp.escape(cask_path("local-caffeine").read)}/m).to_stdout .and not_to_output.to_stderr .and be_a_success end diff --git a/Library/Homebrew/test/dev-cmd/contributions_spec.rb b/Library/Homebrew/test/dev-cmd/contributions_spec.rb index 766ef467b19d0..af938f65c7895 100644 --- a/Library/Homebrew/test/dev-cmd/contributions_spec.rb +++ b/Library/Homebrew/test/dev-cmd/contributions_spec.rb @@ -10,6 +10,7 @@ before { stub_const("HOMEBREW_CACHE", mktmpdir) } it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "contributions" it "documents the governance reporting quarters" do help_text = described_class.parser.generate_help_text.gsub(/\s+/, " ") diff --git a/Library/Homebrew/test/dev-cmd/debugger_spec.rb b/Library/Homebrew/test/dev-cmd/debugger_spec.rb new file mode 100644 index 0000000000000..e548b23af4bc7 --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/debugger_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew debugger", type: :system do + it_behaves_like "a documented command", "debugger" +end diff --git a/Library/Homebrew/test/dev-cmd/edit_spec.rb b/Library/Homebrew/test/dev-cmd/edit_spec.rb index 843c91c0ba4ac..530342a975738 100644 --- a/Library/Homebrew/test/dev-cmd/edit_spec.rb +++ b/Library/Homebrew/test/dev-cmd/edit_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Homebrew::DevCmd::Edit do it_behaves_like "parseable arguments" - it "opens a given Formula in an editor", :integration_test do + it "opens a given Formula and prints a Cask's path", :cask, :integration_test do HOMEBREW_REPOSITORY.cd do system "git", "init" end @@ -18,6 +18,10 @@ .to output(/# something here/).to_stdout .and not_to_output.to_stderr .and be_a_success + expect { brew "edit", "--print-path", "--cask", cask_path("local-caffeine") } + .to output("#{cask_path("local-caffeine")}\n").to_stdout + .and not_to_output.to_stderr + .and be_a_success end it "auto-taps core when editing an API-known formula without the tap installed" do diff --git a/Library/Homebrew/test/dev-cmd/generate-man-completions_spec.rb b/Library/Homebrew/test/dev-cmd/generate-man-completions_spec.rb index 602f213864c88..f4e17ba70c71d 100644 --- a/Library/Homebrew/test/dev-cmd/generate-man-completions_spec.rb +++ b/Library/Homebrew/test/dev-cmd/generate-man-completions_spec.rb @@ -6,4 +6,5 @@ RSpec.describe Homebrew::DevCmd::GenerateManCompletions do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "generate-man-completions" end diff --git a/Library/Homebrew/test/dev-cmd/generate-zap_spec.rb b/Library/Homebrew/test/dev-cmd/generate-zap_spec.rb index 7da1f79d72d27..6abd612d185d3 100644 --- a/Library/Homebrew/test/dev-cmd/generate-zap_spec.rb +++ b/Library/Homebrew/test/dev-cmd/generate-zap_spec.rb @@ -9,6 +9,24 @@ it_behaves_like "parseable arguments" + it "generates a zap stanza for a Cask", :cask, :integration_test do + caskfile = CoreCaskTap.instance.cask_dir/"zap-integration.rb" + caskfile.write <<~RUBY + cask "zap-integration" do + version "1.0" + sha256 :no_check + url "https://brew.sh/zap-integration-1.0.zip" + app "CodexZapCoverageRandom.app" + end + RUBY + CoreCaskTap.instance.clear_cache + + expect { brew "generate-zap", caskfile } + .to output(/Scanning for files matching.*No zap stanza required/m).to_stdout + .and output(/No files found matching/).to_stderr + .and be_a_success + end + describe "#run" do it "surfaces Full Disk Access guidance when scanning raises a permission error" do generate_zap = described_class.new(["--name", "Test"]) diff --git a/Library/Homebrew/test/dev-cmd/install-bundler-gems_spec.rb b/Library/Homebrew/test/dev-cmd/install-bundler-gems_spec.rb index 68cc17a04d1cb..864bf1923296c 100644 --- a/Library/Homebrew/test/dev-cmd/install-bundler-gems_spec.rb +++ b/Library/Homebrew/test/dev-cmd/install-bundler-gems_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Homebrew::DevCmd::InstallBundlerGems do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "install-bundler-gems" it "resets Bootsnap after cleaning gems" do bundle = "/tmp/bundle" diff --git a/Library/Homebrew/test/dev-cmd/lgtm_spec.rb b/Library/Homebrew/test/dev-cmd/lgtm_spec.rb index c28dfbaf45273..054b22dc1122c 100644 --- a/Library/Homebrew/test/dev-cmd/lgtm_spec.rb +++ b/Library/Homebrew/test/dev-cmd/lgtm_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Homebrew::DevCmd::Lgtm do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "lgtm" describe "#run" do subject(:lgtm) { described_class.new(args) } diff --git a/Library/Homebrew/test/dev-cmd/linkage_spec.rb b/Library/Homebrew/test/dev-cmd/linkage_spec.rb index bdd75d2caccad..b8f2cd3a90520 100644 --- a/Library/Homebrew/test/dev-cmd/linkage_spec.rb +++ b/Library/Homebrew/test/dev-cmd/linkage_spec.rb @@ -7,11 +7,17 @@ RSpec.describe Homebrew::DevCmd::Linkage do it_behaves_like "parseable arguments" - it "works when no arguments are provided", :integration_test do + it "checks installed Formulae", :integration_test do + setup_test_formula "testball", tab_attributes: { installed_on_request: true } + expect { brew "linkage" } .to be_a_success .and not_to_output.to_stdout .and not_to_output.to_stderr + expect { brew "linkage", "testball" } + .to be_a_success + .and not_to_output.to_stdout + .and not_to_output.to_stderr end it "accepts no_linkage dependency tag" do diff --git a/Library/Homebrew/test/dev-cmd/livecheck_spec.rb b/Library/Homebrew/test/dev-cmd/livecheck_spec.rb index 564c6806d0e6a..6bb32dc8d00bc 100644 --- a/Library/Homebrew/test/dev-cmd/livecheck_spec.rb +++ b/Library/Homebrew/test/dev-cmd/livecheck_spec.rb @@ -7,16 +7,20 @@ RSpec.describe Homebrew::DevCmd::LivecheckCmd do it_behaves_like "parseable arguments" - it "reports the latest version of a Formula", :integration_test, :needs_network do + it "skips a Formula and Cask with disabled livechecks", :cask, :integration_test do content = <<~RUBY desc "Some test" homepage "https://github.com/Homebrew/brew" url "https://brew.sh/test-1.0.0.tgz" + + livecheck do + skip "integration test" + end RUBY setup_test_formula("test", content) - expect { brew "livecheck", "test" } - .to output(/test: /).to_stdout + expect { brew "livecheck", "test", "latest-with-livecheck-skip" } + .to output(/latest-with-livecheck-skip: skipped.*test: skipped - integration test/m).to_stdout .and not_to_output.to_stderr .and be_a_success end diff --git a/Library/Homebrew/test/dev-cmd/rubocop_spec.rb b/Library/Homebrew/test/dev-cmd/rubocop_spec.rb new file mode 100644 index 0000000000000..b78cec9a37202 --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/rubocop_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew rubocop", type: :system do + it_behaves_like "a documented command", "rubocop", shell: true +end diff --git a/Library/Homebrew/test/dev-cmd/rubydoc_spec.rb b/Library/Homebrew/test/dev-cmd/rubydoc_spec.rb new file mode 100644 index 0000000000000..f5efb90576f5b --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/rubydoc_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew rubydoc", type: :system do + it_behaves_like "a documented command", "rubydoc" +end diff --git a/Library/Homebrew/test/dev-cmd/style_spec.rb b/Library/Homebrew/test/dev-cmd/style_spec.rb index e47b7bf9d0669..08adf7638f239 100644 --- a/Library/Homebrew/test/dev-cmd/style_spec.rb +++ b/Library/Homebrew/test/dev-cmd/style_spec.rb @@ -6,4 +6,20 @@ RSpec.describe Homebrew::DevCmd::StyleCmd do it_behaves_like "parseable arguments" + + it "checks a Formula and Cask", :cask, :integration_test do + formula_file = setup_test_formula "testball" + HOMEBREW_LIBRARY.mkpath + FileUtils.ln_s HOMEBREW_LIBRARY_PATH.parent/".rubocop.yml", HOMEBREW_LIBRARY/".rubocop.yml" + FileUtils.ln_s HOMEBREW_LIBRARY_PATH, HOMEBREW_LIBRARY/"Homebrew" + + begin + expect do + brew "style", "--only-cops=Layout/TrailingWhitespace", formula_file, cask_path("local-caffeine") + end.to be_a_success + ensure + FileUtils.rm_f HOMEBREW_LIBRARY/".rubocop.yml" + FileUtils.rm_f HOMEBREW_LIBRARY/"Homebrew" + end + end end diff --git a/Library/Homebrew/test/dev-cmd/test-bot_spec.rb b/Library/Homebrew/test/dev-cmd/test-bot_spec.rb new file mode 100644 index 0000000000000..1534c796eebf8 --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/test-bot_spec.rb @@ -0,0 +1,6 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe "brew test-bot", type: :system do + it_behaves_like "a documented command", "test-bot" +end diff --git a/Library/Homebrew/test/dev-cmd/tests_spec.rb b/Library/Homebrew/test/dev-cmd/tests_spec.rb index dca023772d3f9..ca9abfe092373 100644 --- a/Library/Homebrew/test/dev-cmd/tests_spec.rb +++ b/Library/Homebrew/test/dev-cmd/tests_spec.rb @@ -7,6 +7,7 @@ RSpec.describe Homebrew::DevCmd::Tests do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "tests" describe "#run" do subject(:tests) { described_class.new(args) } diff --git a/Library/Homebrew/test/dev-cmd/typecheck_spec.rb b/Library/Homebrew/test/dev-cmd/typecheck_spec.rb index b6d953b6650da..0effcde420a57 100644 --- a/Library/Homebrew/test/dev-cmd/typecheck_spec.rb +++ b/Library/Homebrew/test/dev-cmd/typecheck_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Homebrew::DevCmd::Typecheck do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "typecheck" describe "#trim_rubocop_rbi" do let(:rbi_file) { Pathname.new("#{TEST_FIXTURE_DIR}/rubocop@x.x.x.rbi") } diff --git a/Library/Homebrew/test/dev-cmd/unpack_spec.rb b/Library/Homebrew/test/dev-cmd/unpack_spec.rb index 7ece5ddfbb932..6a06c945fb728 100644 --- a/Library/Homebrew/test/dev-cmd/unpack_spec.rb +++ b/Library/Homebrew/test/dev-cmd/unpack_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Homebrew::DevCmd::Unpack do it_behaves_like "parseable arguments" - it "unpacks a given Formula's archive", :integration_test do + it "unpacks a given Formula and Cask archive", :cask, :integration_test do setup_test_formula "testball" mktmpdir do |path| @@ -16,16 +16,12 @@ expect(path/"testball-0.1").to be_a_directory end - end - - it "unpacks a given Cask's archive" do - caffeine_cask = Cask::CaskLoader.load(cask_path("local-caffeine")) mktmpdir do |path| - expect { described_class.new([cask_path("local-caffeine").to_s, "--destdir=#{path}"]).run } - .not_to raise_error + expect { brew "unpack", "--cask", cask_path("local-caffeine"), "--destdir=#{path}" } + .to be_a_success - expect(path/"local-caffeine-#{caffeine_cask.version}").to be_a_directory + expect(path/"local-caffeine-1.2.3").to be_a_directory end end end diff --git a/Library/Homebrew/test/dev-cmd/update-perl-resources_spec.rb b/Library/Homebrew/test/dev-cmd/update-perl-resources_spec.rb index 11e3dfb428e36..0ad505c2dd125 100644 --- a/Library/Homebrew/test/dev-cmd/update-perl-resources_spec.rb +++ b/Library/Homebrew/test/dev-cmd/update-perl-resources_spec.rb @@ -6,4 +6,12 @@ RSpec.describe Homebrew::DevCmd::UpdatePerlResources do it_behaves_like "parseable arguments" + + it "loads a Formula's Perl resources", :integration_test do + setup_test_formula "testball" + + expect { brew "update-perl-resources", "--print-only", "testball" } + .to output(/"testball" has no CPAN resources to update/).to_stderr + .and be_a_failure + end end diff --git a/Library/Homebrew/test/dev-cmd/update-python-resources_spec.rb b/Library/Homebrew/test/dev-cmd/update-python-resources_spec.rb index e5ce46b18c8b8..55d32043c8704 100644 --- a/Library/Homebrew/test/dev-cmd/update-python-resources_spec.rb +++ b/Library/Homebrew/test/dev-cmd/update-python-resources_spec.rb @@ -6,4 +6,15 @@ RSpec.describe Homebrew::DevCmd::UpdatePythonResources do it_behaves_like "parseable arguments" + + it "loads a Formula's Python resources", :integration_test do + setup_test_formula "testball" + + expect do + brew "update-python-resources", "--print-only", "--version=0.2", + "--ignore-non-pypi-packages", "testball" + end + .to not_to_output.to_stderr + .and be_a_success + end end diff --git a/Library/Homebrew/test/dev-cmd/update-test_spec.rb b/Library/Homebrew/test/dev-cmd/update-test_spec.rb index 9a5f08040762e..f1d9e5f959752 100644 --- a/Library/Homebrew/test/dev-cmd/update-test_spec.rb +++ b/Library/Homebrew/test/dev-cmd/update-test_spec.rb @@ -6,4 +6,5 @@ RSpec.describe Homebrew::DevCmd::UpdateTest do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "update-test" end diff --git a/Library/Homebrew/test/dev-cmd/vendor-gems_spec.rb b/Library/Homebrew/test/dev-cmd/vendor-gems_spec.rb index 04f7b41f0f5df..50edc340e6c63 100644 --- a/Library/Homebrew/test/dev-cmd/vendor-gems_spec.rb +++ b/Library/Homebrew/test/dev-cmd/vendor-gems_spec.rb @@ -6,6 +6,7 @@ RSpec.describe Homebrew::DevCmd::VendorGems do it_behaves_like "parseable arguments" + it_behaves_like "a documented command", "vendor-gems" sig { returns(Homebrew::DevCmd::VendorGems) } def vendor_gems_command diff --git a/Library/Homebrew/test/dev-cmd/verify_spec.rb b/Library/Homebrew/test/dev-cmd/verify_spec.rb index a554da52de7d6..2631d56c9eec9 100644 --- a/Library/Homebrew/test/dev-cmd/verify_spec.rb +++ b/Library/Homebrew/test/dev-cmd/verify_spec.rb @@ -6,4 +6,12 @@ RSpec.describe Homebrew::DevCmd::Verify do it_behaves_like "parseable arguments" + + it "checks whether a Formula has a bottle to verify", :integration_test do + setup_test_formula "testball" + + expect { brew "verify", "testball" } + .to output(/Bottle for tag .* is unavailable\./).to_stderr + .and be_a_success + end end diff --git a/Library/Homebrew/test/manpages_spec.rb b/Library/Homebrew/test/manpages_spec.rb index ed176fdf2279f..1ef57e1fd69fd 100644 --- a/Library/Homebrew/test/manpages_spec.rb +++ b/Library/Homebrew/test/manpages_spec.rb @@ -72,4 +72,18 @@ def subcommand_parser expect(manpage).not_to include(*hidden_commands) end + + it "has integration test coverage for every documented command" do + missing_commands = { + "cmd" => Commands.internal_commands, + "dev-cmd" => Commands.internal_developer_commands, + }.flat_map do |directory, commands| + commands.reject do |command| + spec = HOMEBREW_LIBRARY_PATH/"test/#{directory}/#{command}_spec.rb" + spec.exist? && spec.read.match?(/:integration_test|a documented command/) + end + end + + expect(missing_commands).to be_empty + end end diff --git a/Library/Homebrew/test/spec_helper.rb b/Library/Homebrew/test/spec_helper.rb index 5172d3e76f7ce..1cb9c2a46283c 100644 --- a/Library/Homebrew/test/spec_helper.rb +++ b/Library/Homebrew/test/spec_helper.rb @@ -5,6 +5,8 @@ require "simplecov" require "simplecov-cobertura" SimpleCov.start + require_relative "support/helper/integration_coverage" + Homebrew::TestIntegrationCoverage.prepare! formatters = [ SimpleCov::Formatter::HTMLFormatter, diff --git a/Library/Homebrew/test/support/helper/integration_coverage.rb b/Library/Homebrew/test/support/helper/integration_coverage.rb new file mode 100644 index 0000000000000..a5bc78956b052 --- /dev/null +++ b/Library/Homebrew/test/support/helper/integration_coverage.rb @@ -0,0 +1,63 @@ +# typed: false +# frozen_string_literal: true + +require "coverage" +require "json" + +module Homebrew + module TestIntegrationCoverage + DIRECTORY_ENV = "HOMEBREW_INTEGRATION_COVERAGE_DIR" + private_constant :DIRECTORY_ENV + + class << self + def prepare! + require "fileutils" + + directory = File.join( + SimpleCov.coverage_path, + ".integration#{ENV.fetch("TEST_ENV_NUMBER", "")}", + ) + FileUtils.rm_rf directory + FileUtils.mkdir_p directory + ENV[DIRECTORY_ENV] = directory + Kernel.at_exit { store_results!(directory) } + end + + def start! + Coverage.start(lines: true, branches: true, eval: true) + Kernel.at_exit { dump! } + end + + def dump! + return unless Coverage.running? + + directory = ENV.fetch(DIRECTORY_ENV) + result_path = File.join(directory, "#{ENV.fetch("HOMEBREW_INTEGRATION_TEST")}.json") + temporary_path = "#{result_path}.#{Process.pid}.tmp" + File.write(temporary_path, JSON.generate(Coverage.result)) + File.rename(temporary_path, result_path) + end + + def store_results!(directory) + accumulator = SimpleCov::Combine::CoverageAccumulator.new + Dir.glob(File.join(directory, "*.json")).each do |path| + coverage = JSON.parse(File.read(path)) + coverage = SimpleCov::UselessResultsRemover.call(coverage) + accumulator.absorb(coverage) + end + result = accumulator.result + return if result.nil? + + command_name = "brew_i:#{Process.pid}:#{ENV.fetch("TEST_ENV_NUMBER", "")}" + SimpleCov::ResultMerger.store_result(SimpleCov::Result.new(result, command_name:)) + ensure + require "fileutils" + + FileUtils.rm_rf directory + ENV.delete(DIRECTORY_ENV) if ENV[DIRECTORY_ENV] == directory + end + end + end +end + +Homebrew::TestIntegrationCoverage.start! if ENV["HOMEBREW_INTEGRATION_TEST"] diff --git a/Library/Homebrew/test/support/helper/integration_coverage_spec.rb b/Library/Homebrew/test/support/helper/integration_coverage_spec.rb new file mode 100644 index 0000000000000..7a8b9d8577d59 --- /dev/null +++ b/Library/Homebrew/test/support/helper/integration_coverage_spec.rb @@ -0,0 +1,55 @@ +# typed: true +# frozen_string_literal: true + +require "test/support/helper/integration_coverage" + +RSpec.describe Homebrew::TestIntegrationCoverage do + it "records coverage before an integration command execs" do + mktmpdir do |directory| + env = { + "HOMEBREW_INTEGRATION_COVERAGE_DIR" => directory.to_s, + "HOMEBREW_INTEGRATION_TEST" => "integration-command", + "HOMEBREW_TESTS_COVERAGE" => "1", + } + coverage_helper = HOMEBREW_LIBRARY_PATH/"test/support/helper/integration_coverage" + integration_mocks = HOMEBREW_LIBRARY_PATH/"test/support/helper/integration_mocks" + script = "abort if defined?(SimpleCov); Object.new.extend(Homebrew).exec('/usr/bin/true')" + + _, _, status = Open3.capture3( + env, + *HOMEBREW_RUBY_EXEC_ARGS, + "-r#{coverage_helper}", + "-r#{integration_mocks}", + "-e", script + ) + result_path = directory/"integration-command.json" + coverage = result_path.exist? ? JSON.parse(result_path.read) : {} + + expect([status.success?, coverage.key?(integration_mocks.sub_ext(".rb").to_s)]).to eq([true, true]) + end + end + + it "merges child results before storing them with SimpleCov" do + require "simplecov" + + mktmpdir do |directory| + source = (HOMEBREW_LIBRARY_PATH/"brew.rb").to_s + branches = { + [:if, 0, 1, 0, 1, 1] => { + [:then, 1, 1, 0, 1, 1] => 1, + }, + } + File.write(directory/"one.json", JSON.generate(source => { lines: [1], branches: })) + File.write(directory/"two.json", JSON.generate(source => { lines: [2], branches: })) + stored_results = [] + allow(SimpleCov::ResultMerger).to receive(:store_result) { |result| stored_results << result } + + described_class.store_results!(directory) + + stored_result = stored_results.fetch(0) + coverage = stored_result.original_result.fetch(source) + expect([coverage.fetch("lines"), coverage.fetch("branches").values.first.values]) + .to eq([[3], [2]]) + end + end +end diff --git a/Library/Homebrew/test/support/helper/integration_mocks.rb b/Library/Homebrew/test/support/helper/integration_mocks.rb index 832a55178c52b..16a5d1014e212 100644 --- a/Library/Homebrew/test/support/helper/integration_mocks.rb +++ b/Library/Homebrew/test/support/helper/integration_mocks.rb @@ -13,7 +13,7 @@ def check_integration_test def exec(*args) if ENV["HOMEBREW_TESTS_COVERAGE"] && ENV["HOMEBREW_INTEGRATION_TEST"] # Ensure we get coverage results before replacing the current process. - SimpleCov.result + Homebrew::TestIntegrationCoverage.dump! end Kernel.exec(*args) end diff --git a/Library/Homebrew/test/support/helper/simplecov_start.rb b/Library/Homebrew/test/support/helper/simplecov_start.rb deleted file mode 100644 index 99331c3dcb00a..0000000000000 --- a/Library/Homebrew/test/support/helper/simplecov_start.rb +++ /dev/null @@ -1,9 +0,0 @@ -# typed: strict -# frozen_string_literal: true - -require "rubygems/version" - -simplecov_root = File.expand_path("../../..", __dir__) -Dir.chdir(simplecov_root) { require "simplecov" } - -SimpleCov.start diff --git a/Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb b/Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb index ff4a8df6b68b7..29aa99e0f9018 100644 --- a/Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb +++ b/Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb @@ -44,6 +44,28 @@ def expects_call_stack_jump? RSpec::Matchers.define_negated_matcher :be_a_failure, :be_a_success +RSpec.shared_examples "a documented command" do |command, shell: false| + T.bind(self, T.class_of(RSpec::Core::ExampleGroup)) + + it "shows its help", :integration_test, + documented_command: T.cast(command, String), documented_command_shell: T.cast(shell, T::Boolean) do + T.bind(self, RSpec::Core::ExampleGroup) + example = RSpec.current_example + raise "Current RSpec example is unavailable" if example.nil? + + documented_command = T.cast(example.metadata.fetch(:documented_command), String) + documented_command_shell = T.cast(example.metadata.fetch(:documented_command_shell), T::Boolean) + + expect do + if documented_command_shell + T.unsafe(self).brew_sh("help", documented_command) + else + T.unsafe(self).brew("help", documented_command) + end + end.to be_a_success + end +end + module Test module Helper module IntegrationTest @@ -75,6 +97,7 @@ def brew(*args) ].compact.join(File::PATH_SEPARATOR) env["HOMEBREW_AVOID_NESTED_SANDBOXING"] = "1" + env["HOMEBREW_INTEGRATION_COVERAGE_DIR"] = ENV.fetch("HOMEBREW_INTEGRATION_COVERAGE_DIR", nil) env.merge!( "PATH" => path, "HOMEBREW_PATH" => path, @@ -85,26 +108,15 @@ def brew(*args) "HOMEBREW_ASK" => nil, "HOMEBREW_USE_RUBY_FROM_PATH" => ENV.fetch("HOMEBREW_USE_RUBY_FROM_PATH", nil), "HOMEBREW_NO_INSTALL_FROM_API" => ENV.fetch("HOMEBREW_NO_INSTALL_FROM_API", nil), + "HOMEBREW_SORBET_RUNTIME" => nil, + "HOMEBREW_SORBET_RECURSIVE" => nil, "GEM_HOME" => nil, ) @ruby_args ||= begin ruby_args = HOMEBREW_RUBY_EXEC_ARGS.dup if ENV["HOMEBREW_TESTS_COVERAGE"] - simplecov_spec = Gem.loaded_specs["simplecov"] - parallel_tests_spec = Gem.loaded_specs["parallel_tests"] - specs = T.let([], T::Array[Gem::Specification]) - [simplecov_spec, parallel_tests_spec].each do |spec| - specs << spec - spec.runtime_dependencies.each do |dep| - specs += dep.to_specs - rescue Gem::LoadError => e - T.bind(self, Utils::Output::Mixin) - onoe e - end - end - specs.flat_map(&:full_require_paths).each { |lib| ruby_args << "-I" << lib } - ruby_args << "-r#{HOMEBREW_LIBRARY_PATH}/test/support/helper/simplecov_start" + ruby_args << "-r#{HOMEBREW_LIBRARY_PATH}/test/support/helper/integration_coverage" end ruby_args << "-r#{HOMEBREW_LIBRARY_PATH}/test/support/helper/integration_mocks" ruby_args << "-e" << "$0 = ARGV.shift; load($0)" @@ -216,11 +228,11 @@ class #{Formulary.class_s(name)} < Formula return formula_path if tab_attributes.nil? - keg = ::Formula[name].prefix + formula = ::Formula[name] + keg = formula.prefix keg.mkpath - tab = Tab.for_name(name) - tab.tabfile ||= keg/AbstractTab::FILENAME + tab = Tab.create(formula) tab_attributes.each do |key, value| tab.public_send(:"#{key}=", value) end @@ -263,25 +275,6 @@ def setup_test_tap path end - sig { params(old_name: String, new_name: String).void } - def install_and_rename_coretap_formula(old_name, new_name) - CoreTap.instance.path.cd do |tap_path| - system "git", "init" - system "git", "add", "--all" - system "git", "commit", "-m", - "#{old_name.capitalize} has not yet been renamed" - - brew "install", old_name - - (tap_path/"Formula/#{old_name}.rb").unlink - (tap_path/"formula_renames.json").write JSON.pretty_generate(old_name => new_name) - - system "git", "add", "--all" - system "git", "commit", "-m", - "#{old_name.capitalize} has been renamed to #{new_name.capitalize}" - end - end - sig { returns(String) } def testball "#{TEST_FIXTURE_DIR}/testball.rb" diff --git a/Library/Homebrew/test/support/helper/spec/shared_context/integration_test_spec.rb b/Library/Homebrew/test/support/helper/spec/shared_context/integration_test_spec.rb new file mode 100644 index 0000000000000..b73037d1a6d03 --- /dev/null +++ b/Library/Homebrew/test/support/helper/spec/shared_context/integration_test_spec.rb @@ -0,0 +1,50 @@ +# typed: true +# frozen_string_literal: true + +RSpec.describe Test::Helper::IntegrationTest do + subject(:helper) { Object.new.extend(described_class) } + + let(:status) { instance_double(Process::Status) } + + before do + allow(helper).to receive(:command_id).and_return("integration-command") + end + + around do |example| + (HOMEBREW_PREFIX/"bin").mkpath + example.run + ensure + FileUtils.rm_rf HOMEBREW_PREFIX/"bin" + end + + it "disables Sorbet checking in Ruby integration commands" do + captured_env = {} + allow(Open3).to receive(:capture3) do |env, *_args| + captured_env.replace(env) + ["", "", status] + end + + helper.brew "help" + + expect(captured_env.slice("HOMEBREW_SORBET_RUNTIME", "HOMEBREW_SORBET_RECURSIVE")).to eq( + "HOMEBREW_SORBET_RUNTIME" => nil, + "HOMEBREW_SORBET_RECURSIVE" => nil, + ) + end + + it "uses the lightweight integration coverage recorder" do + ENV["HOMEBREW_TESTS_COVERAGE"] = "1" + captured_command = [] + allow(Open3).to receive(:capture3) do |_env, *command| + captured_command.replace(command.map(&:to_s)) + ["", "", status] + end + + helper.brew "help" + + expect([ + captured_command.any? { |arg| arg.include?("integration_coverage") }, + captured_command.any? { |arg| arg.include?("simplecov_start") }, + ]).to eq([true, false]) + end +end