From ef815369e2f95acd913b7ba7f1fa07af23513978 Mon Sep 17 00:00:00 2001 From: Florian Weikert Date: Mon, 20 Mar 2023 13:39:03 +0100 Subject: [PATCH 01/22] Fix style issue (#439) --- core/core.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/core.go b/core/core.go index 01f5f685..a4388133 100644 --- a/core/core.go +++ b/core/core.go @@ -604,7 +604,7 @@ func insertArgs(baseArgs []string, newArgs []string) []string { func parseStartupOptions(baseArgs []string) []string { var result []string - var BAZEL_COMMANDS = map[string]bool{ + var bazelCommands = map[string]bool{ "analyze-profile": true, "aquery": true, "build": true, @@ -629,7 +629,7 @@ func parseStartupOptions(baseArgs []string) []string { } // Arguments before a Bazel command are startup options. for _, arg := range baseArgs { - if _, ok := BAZEL_COMMANDS[arg]; ok { + if _, ok := bazelCommands[arg]; ok { return result } result = append(result, arg) From 204a69f64872c87faa011a02b4a953a19c0693f4 Mon Sep 17 00:00:00 2001 From: Florian Weikert Date: Wed, 22 Mar 2023 16:20:34 +0100 Subject: [PATCH 02/22] Stop downloading existing Bazel binaries (#438) Due to a bug Bazelisk never read any binaries from the cache, thus resulting in severe runtime penalties. --- core/core.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/core/core.go b/core/core.go index a4388133..c4dccf99 100644 --- a/core/core.go +++ b/core/core.go @@ -106,7 +106,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error } baseDirectory := filepath.Join(bazeliskHome, "downloads", bazelForkOrURL) - bazelPath, err = downloadBazel(resolvedBazelVersion, baseDirectory, repos, downloader) + bazelPath, err = downloadBazelIfNecessary(resolvedBazelVersion, baseDirectory, repos, downloader) if err != nil { return -1, fmt.Errorf("could not download Bazel: %v", err) } @@ -404,20 +404,25 @@ func parseBazelForkAndVersion(bazelForkAndVersion string) (string, string, error return bazelFork, bazelVersion, nil } -func downloadBazel(version string, baseDirectory string, repos *Repositories, downloader DownloadFunc) (string, error) { +func downloadBazelIfNecessary(version string, baseDirectory string, repos *Repositories, downloader DownloadFunc) (string, error) { pathSegment, err := platforms.DetermineBazelFilename(version, false) if err != nil { return "", fmt.Errorf("could not determine path segment to use for Bazel binary: %v", err) } + destDir := filepath.Join(baseDirectory, pathSegment, "bin") destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() - destinationDir := filepath.Join(baseDirectory, pathSegment, "bin") + + destPath := filepath.Join(destDir, destFile) + if _, err := os.Stat(destPath); err == nil { + return destPath, nil + } if url := GetEnvOrConfig(BaseURLEnv); url != "" { - return repos.DownloadFromBaseURL(url, version, destinationDir, destFile) + return repos.DownloadFromBaseURL(url, version, destDir, destFile) } - return downloader(destinationDir, destFile) + return downloader(destDir, destFile) } func copyFile(src, dst string, perm os.FileMode) error { From b76d71d205223e7485c5901bfc54649ab3ae1f50 Mon Sep 17 00:00:00 2001 From: Julio Merino Date: Wed, 5 Apr 2023 09:06:49 -0700 Subject: [PATCH 03/22] Implement support for BAZELISK_VERIFY_SHA256 (#441) The new BAZELISK_VERIFY_SHA256 variable can be set to the expected SHA256 hash of the downloaded Bazel binary. If set, then the binary is required to match the hash before it is used. This is important for cases where provenance of the artifact cannot be asserted purely via the HTTPS trust chain (such as what happens in a mutable artifact repository with lax access controls). --- README.md | 3 ++- bazelisk_test.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ core/core.go | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 93f63708..1d059b51 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Additionally, a few special version names are supported for our official release ## Where does Bazelisk get Bazel from? -By default Bazelisk retrieves Bazel releases, release candidates and binaries built at green commits from Google Cloud Storage. +By default Bazelisk retrieves Bazel releases, release candidates and binaries built at green commits from Google Cloud Storage. The downloaded artifacts are validated against the SHA256 value recorded in `BAZELISK_VERIFY_SHA256` if this variable is set in the configuration file. As mentioned in the previous section, the `/` version format allows you to use your own Bazel fork hosted on GitHub: @@ -149,6 +149,7 @@ The following variables can be set: - `BAZELISK_SHUTDOWN` - `BAZELISK_SKIP_WRAPPER` - `BAZELISK_USER_AGENT` +- `BAZELISK_VERIFY_SHA256` - `USE_BAZEL_VERSION` Configuration variables are evaluated with precedence order. The preferred values are derived in order from highest to lowest precedence as follows: diff --git a/bazelisk_test.sh b/bazelisk_test.sh index ad99dbe1..66c5b7b0 100755 --- a/bazelisk_test.sh +++ b/bazelisk_test.sh @@ -320,6 +320,50 @@ function test_bazel_download_path_go() { (echo "FAIL: Expected to download bazel binary into specific path."; exit 1) } +function test_bazel_verify_sha256() { + setup + + echo "6.1.1" > .bazelversion + + # First try to download and expect an invalid hash (it doesn't matter what it is). + if BAZELISK_HOME="$BAZELISK_HOME" BAZELISK_VERIFY_SHA256="invalid-hash" \ + bazelisk version 2>&1 | tee log; then + echo "FAIL: Command should have errored out"; exit 1 + fi + + grep "need sha256=invalid-hash" log || \ + (echo "FAIL: Expected to find hash mismatch"; exit 1) + + # IMPORTANT: The mixture of lowercase and uppercase letters in the hashes below is + # intentional to ensure the variable contents are normalized before comparison. + # If updating these values, re-introduce randomness. + local os="$(uname -s | tr A-Z a-z)" + case "${os}" in + darwin) + expected_sha256="038e95BAE998340812562ab8d6ada1a187729630bc4940a4cd7920cc78acf156" + ;; + linux) + expected_sha256="651a20d85531325df406b38f38A1c2578c49D5e61128fba034f5b6abdb3d303f" + ;; + msys*|mingw*|cygwin*) + expected_sha256="1d997D344936a1d98784ae58db1152d083569556f85cd845e6e340EE855357f9" + ;; + *) + echo "FAIL: Unknown OS ${os} in test" + exit 1 + ;; + esac + + # Now try the same download as before but with the correct hash expectation. Note that the + # hash has a random uppercase / lowercase mixture to ensure this does not impact equality + # checks. + BAZELISK_HOME="$BAZELISK_HOME" BAZELISK_VERIFY_SHA256="${expected_sha256}" \ + bazelisk version 2>&1 | tee log + + grep "Build label:" log || \ + (echo "FAIL: Expected to find 'Build label' in the output of 'bazelisk version'"; exit 1) +} + function test_bazel_download_path_py() { setup @@ -411,6 +455,10 @@ if [[ $BAZELISK_VERSION == "GO" ]]; then test_bazel_download_path_go echo + echo '# test_bazel_verify_sha256' + test_bazel_verify_sha256 + echo + echo "# test_bazel_prepend_binary_directory_to_path_go" test_bazel_prepend_binary_directory_to_path_go echo diff --git a/core/core.go b/core/core.go index c4dccf99..139c17b9 100644 --- a/core/core.go +++ b/core/core.go @@ -411,6 +411,9 @@ func downloadBazelIfNecessary(version string, baseDirectory string, repos *Repos } destDir := filepath.Join(baseDirectory, pathSegment, "bin") + expectedSha256 := strings.ToLower(GetEnvOrConfig("BAZELISK_VERIFY_SHA256")) + + tmpDestFile := "bazel-tmp" + platforms.DetermineExecutableFilenameSuffix() destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() destPath := filepath.Join(destDir, destFile) @@ -418,11 +421,45 @@ func downloadBazelIfNecessary(version string, baseDirectory string, repos *Repos return destPath, nil } + var tmpDestPath string if url := GetEnvOrConfig(BaseURLEnv); url != "" { - return repos.DownloadFromBaseURL(url, version, destDir, destFile) + tmpDestPath, err = repos.DownloadFromBaseURL(url, version, destDir, tmpDestFile) + } else { + tmpDestPath, err = downloader(destDir, tmpDestFile) + } + if err != nil { + return "", err } - return downloader(destDir, destFile) + if len(expectedSha256) > 0 { + f, err := os.Open(tmpDestPath) + if err != nil { + os.Remove(tmpDestPath) + return "", fmt.Errorf("cannot open %s after download: %v", tmpDestPath, err) + } + defer os.Remove(tmpDestPath) + // We cannot defer f.Close() because keeping the handle open when we try to do the + // rename later on fails on Windows. + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + f.Close() + return "", fmt.Errorf("cannot compute sha256 of %s after download: %v", tmpDestPath, err) + } + f.Close() + + actualSha256 := strings.ToLower(fmt.Sprintf("%x", h.Sum(nil))) + if expectedSha256 != actualSha256 { + return "", fmt.Errorf("%s has sha256=%s but need sha256=%s", tmpDestPath, actualSha256, expectedSha256) + } + } + + // Only place the downloaded binary in its final location once we know it is fully downloaded + // and valid, to prevent invalid files from ever being executed. + if err = os.Rename(tmpDestPath, destPath); err != nil { + return "", fmt.Errorf("cannot rename %s to %s: %v", tmpDestPath, destPath, err) + } + return destPath, nil } func copyFile(src, dst string, perm os.FileMode) error { From 371319499f90dc37afd01e7d4a3bab90a056e6c0 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Fri, 28 Apr 2023 11:22:07 +0200 Subject: [PATCH 04/22] Support bisecting Bazel to find which Bazel change breaks your build (#451) * Support bisecting Bazel to find which Bazel change breaks your build - Use --bisect=.. to specify the bisecting range, Bazelisk uses the GitHub API to get the list of commits to bisect. You may need to set `BAZELISK_GITHUB_TOKEN` to get around GitHub rate limit. - BAZELISK_SHUTDOWN, BAZELISK_CLEAN can be used to run `bazel shutdown` or `bazel clean --expunge` between builds. * Add doc * Remove unused struct * small fixes * Fix --strict --- README.md | 23 +++++- core/core.go | 216 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 216 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 1d059b51..76a3010e 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,9 @@ require users update their bazel. [shell wrapper script]: https://github.com/bazelbuild/bazel/blob/master/scripts/packages/bazel.sh ## Other features -The Go version of Bazelisk offers two new flags. +The Go version of Bazelisk offers three new flags. + +### --strict `--strict` expands to the set of incompatible flags which may be enabled for the given version of Bazel. @@ -108,17 +110,32 @@ The Go version of Bazelisk offers two new flags. bazelisk --strict build //... ``` +### --migrate + `--migrate` will run Bazel multiple times to help you identify compatibility issues. If the code fails with `--strict`, the flag `--migrate` will run Bazel with each one of the flag separately, and print a report at the end. This will show you which flags can safely enabled, and which flags require a migration. + +### --bisect + +`--bisect` flag allows you to bisect Bazel versions to find which version introduced a build failure. You can specify the range of versions to bisect with `--bisect=..`, where GOOD is the last known working Bazel version and BAD is the first known non-working Bazel version. Bazelisk uses [GitHub's compare API](https://docs.github.com/en/rest/commits/commits#compare-two-commits) to get the list of commits to bisect. When GOOD is not an ancestor of BAD, GOOD is reset to their merge base commit. + +```shell +bazelisk --bisect=6.0.0..HEAD test //foo:bar_test +``` + +### Useful environment variables + You can set `BAZELISK_INCOMPATIBLE_FLAGS` to set a list of incompatible flags (separated by `,`) to be tested, otherwise Bazelisk tests all flags starting with `--incompatible_`. You can set `BAZELISK_GITHUB_TOKEN` to set a GitHub access token to use for API requests to avoid rate limiting when on shared networks. -You can set `BAZELISK_SHUTDOWN` to run `shutdown` between builds when migrating if you suspect this affects your results. +You can set `BAZELISK_SHUTDOWN` to run `shutdown` between builds when migrating or bisecting if you suspect this affects your results. + +You can set `BAZELISK_CLEAN` to run `clean --expunge` between builds when migrating or bisecting if you suspect this affects your results. -You can set `BAZELISK_CLEAN` to run `clean --expunge` between builds when migrating if you suspect this affects your results. +## tools/bazel If `tools/bazel` exists in your workspace root and is executable, Bazelisk will run this file, instead of the Bazel version it downloaded. It will set the environment variable `BAZEL_REAL` to the path of the downloaded Bazel binary. diff --git a/core/core.go b/core/core.go index 139c17b9..753f88f5 100644 --- a/core/core.go +++ b/core/core.go @@ -6,10 +6,12 @@ package core import ( "bufio" "crypto/sha256" + "encoding/json" "fmt" "io" "io/ioutil" "log" + "net/http" "os" "os/exec" "os/signal" @@ -89,24 +91,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error // If we aren't using a local Bazel binary, we'll have to parse the version string and // download the version that the user wants. if !filepath.IsAbs(bazelPath) { - bazelFork, bazelVersion, err := parseBazelForkAndVersion(bazelVersionString) - if err != nil { - return -1, fmt.Errorf("could not parse Bazel fork and version: %v", err) - } - - var downloader DownloadFunc - resolvedBazelVersion, downloader, err = repos.ResolveVersion(bazeliskHome, bazelFork, bazelVersion) - if err != nil { - return -1, fmt.Errorf("could not resolve the version '%s' to an actual version number: %v", bazelVersion, err) - } - - bazelForkOrURL := dirForURL(GetEnvOrConfig(BaseURLEnv)) - if len(bazelForkOrURL) == 0 { - bazelForkOrURL = bazelFork - } - - baseDirectory := filepath.Join(bazeliskHome, "downloads", bazelForkOrURL) - bazelPath, err = downloadBazelIfNecessary(resolvedBazelVersion, baseDirectory, repos, downloader) + bazelPath, err = downloadBazel(bazelVersionString, bazeliskHome, repos) if err != nil { return -1, fmt.Errorf("could not download Bazel: %v", err) } @@ -130,7 +115,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error return 0, nil } - // --strict and --migrate must be the first argument. + // --strict and --migrate and --bisect must be the first argument. if len(args) > 0 && (args[0] == "--strict" || args[0] == "--migrate") { cmd, err := getBazelCommand(args) if err != nil { @@ -140,7 +125,6 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error if err != nil { return -1, fmt.Errorf("could not get the list of incompatible flags: %v", err) } - if args[0] == "--migrate" { migrate(bazelPath, args[1:], newFlags) } else { @@ -148,6 +132,18 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error // that should be enabled for the given Bazel version. args = insertArgs(args[1:], newFlags) } + } else if len(args) > 0 && strings.HasPrefix(args[0], "--bisect") { + // When --bisect is present, we run the bisect logic. + if !strings.HasPrefix(args[0], "--bisect=") { + return -1, fmt.Errorf("Error: --bisect must have a value. Expected format: '--bisect=..'") + } + value := args[0][len("--bisect="):] + commits := strings.Split(value, "..") + if len(commits) == 2 { + bisect(commits[0], commits[1], args[1:], bazeliskHome, repos) + } else { + return -1, fmt.Errorf("Error: Invalid format for --bisect. Expected format: '--bisect=..'") + } } // print bazelisk version information if "version" is the first argument @@ -404,6 +400,27 @@ func parseBazelForkAndVersion(bazelForkAndVersion string) (string, string, error return bazelFork, bazelVersion, nil } +func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Repositories) (string, error) { + bazelFork, bazelVersion, err := parseBazelForkAndVersion(bazelVersionString) + if err != nil { + return "", fmt.Errorf("could not parse Bazel fork and version: %v", err) + } + + resolvedBazelVersion, downloader, err := repos.ResolveVersion(bazeliskHome, bazelFork, bazelVersion) + if err != nil { + return "", fmt.Errorf("could not resolve the version '%s' to an actual version number: %v", bazelVersion, err) + } + + bazelForkOrURL := dirForURL(GetEnvOrConfig(BaseURLEnv)) + if len(bazelForkOrURL) == 0 { + bazelForkOrURL = bazelFork + } + + baseDirectory := filepath.Join(bazeliskHome, "downloads", bazelForkOrURL) + bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, baseDirectory, repos, downloader) + return bazelPath, err +} + func downloadBazelIfNecessary(version string, baseDirectory string, repos *Repositories, downloader DownloadFunc) (string, error) { pathSegment, err := platforms.DetermineBazelFilename(version, false) if err != nil { @@ -717,6 +734,165 @@ func cleanIfNeeded(bazelPath string, startupOptions []string) { } } +type Commit struct { + SHA string `json:"sha"` +} + +type CompareResponse struct { + Commits []Commit `json:"commits"` + MergeBaseCommit Commit `json:"merge_base_commit"` +} + +func sendRequest(url string) (*http.Response, error) { + client := &http.Client{} + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + githubToken := GetEnvOrConfig("BAZELISK_GITHUB_TOKEN") + if len(githubToken) != 0 { + req.Header.Set("Authorization", fmt.Sprintf("token %s", githubToken)) + } + + return client.Do(req) +} + +func getBazelCommitsBetween(goodCommit string, badCommit string) ([]string, error) { + commitList := make([]string, 0) + page := 1 + perPage := 250 // 250 is the maximum number of commits per page + + for { + url := fmt.Sprintf("https://api.github.com/repos/bazelbuild/bazel/compare/%s...%s?page=%d&per_page=%d", goodCommit, badCommit, page, perPage) + + response, err := sendRequest(url) + if err != nil { + return nil, fmt.Errorf("Error fetching commit data: %v", err) + } + defer response.Body.Close() + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("Error reading response body: %v", err) + } + + if response.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("repository or commit not found: %s", string(body)) + } else if response.StatusCode == 403 { + return nil, fmt.Errorf("github API rate limit hit, consider setting BAZELISK_GITHUB_TOKEN: %s", string(body)) + } else if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected response status code %d: %s", response.StatusCode, string(body)) + } + + var compareResponse CompareResponse + err = json.Unmarshal(body, &compareResponse) + if err != nil { + return nil, fmt.Errorf("Error unmarshaling JSON: %v", err) + } + + if len(compareResponse.Commits) == 0 { + break + } + + mergeBaseCommit := compareResponse.MergeBaseCommit.SHA + if compareResponse.MergeBaseCommit.SHA != goodCommit { + fmt.Printf("The good Bazel commit is not an ancestor of the bad Bazel commit, overriding the good Bazel commit to the merge base commit %s\n", mergeBaseCommit) + goodCommit = mergeBaseCommit + } + + for _, commit := range compareResponse.Commits { + commitList = append(commitList, commit.SHA) + } + + // Check if there are more commits to fetch + if len(compareResponse.Commits) < perPage { + break + } + + page++ + } + + if len(commitList) == 0 { + return nil, fmt.Errorf("no commits found between (%s, %s], the good commit should be first, maybe try with --bisect=%s..%s ?", goodCommit, badCommit, badCommit, goodCommit) + } + fmt.Printf("Found %d commits between (%s, %s]\n", len(commitList), goodCommit, badCommit) + return commitList, nil +} + +func bisect(goodCommit string, badCommit string, args []string, bazeliskHome string, repos *Repositories) { + + // 1. Get the list of commits between goodCommit and badCommit + fmt.Printf("\n\n--- Getting the list of commits between %s and %s\n\n", goodCommit, badCommit) + commitList, err := getBazelCommitsBetween(goodCommit, badCommit) + if err != nil { + log.Fatalf("Failed to get commits: %v", err) + os.Exit(1) + } + + // 2. Check if goodCommit is actually good + fmt.Printf("\n\n--- Verifying if the given good Bazel commit (%s) is actually good\n\n", goodCommit) + bazelExitCode, err := testWithBazelAtCommit(goodCommit, args, bazeliskHome, repos) + if err != nil { + log.Fatalf("could not run Bazel: %v", err) + os.Exit(1) + } + if bazelExitCode != 0 { + fmt.Printf("Failure: Given good bazel commit is already broken.\n") + os.Exit(1) + } + + // 3. Bisect commits + fmt.Printf("\n\n--- Start bisecting\n\n") + left := 0 + right := len(commitList) + for left < right { + mid := (left + right) / 2 + midCommit := commitList[mid] + fmt.Printf("\n\n--- Testing with Bazel built at %s, %d commits remaining...\n\n", midCommit, right -left) + bazelExitCode, err := testWithBazelAtCommit(midCommit, args, bazeliskHome, repos) + if err != nil { + log.Fatalf("could not run Bazel: %v", err) + os.Exit(1) + } + if bazelExitCode == 0 { + fmt.Printf("\n\n--- Succeeded at %s\n\n", midCommit) + left = mid + 1 + } else { + fmt.Printf("\n\n--- Failed at %s\n\n", midCommit) + right = mid + } + } + + // 4. Print the result + fmt.Printf("\n\n--- Bisect Result\n\n") + if right == len(commitList) { + fmt.Printf("first bad commit not found, every commit succeeded.\n") + } else { + firstBadCommit := commitList[right] + fmt.Printf("first bad commit is https://github.com/bazelbuild/bazel/commit/%s\n", firstBadCommit) + } + + os.Exit(0) +} + +func testWithBazelAtCommit(bazelCommit string, args []string, bazeliskHome string, repos *Repositories) (int, error) { + bazelPath, err := downloadBazel(bazelCommit, bazeliskHome, repos) + if err != nil { + return 1, fmt.Errorf("could not download Bazel: %v", err) + } + startupOptions := parseStartupOptions(args) + shutdownIfNeeded(bazelPath, startupOptions) + cleanIfNeeded(bazelPath, startupOptions) + fmt.Printf("bazel %s\n", strings.Join(args, " ")) + bazelExitCode, err := runBazel(bazelPath, args, nil) + if err != nil { + return -1, fmt.Errorf("could not run Bazel: %v", err) + } + return bazelExitCode, nil +} + // migrate will run Bazel with each flag separately and report which ones are failing. func migrate(bazelPath string, baseArgs []string, flags []string) { var startupOptions = parseStartupOptions(baseArgs) From 0d390e803256e6ef26253fa4de41400da25252a3 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Mon, 8 May 2023 14:04:56 +0200 Subject: [PATCH 05/22] --bisect: ignore merge commit (#458) --- core/core.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/core.go b/core/core.go index 753f88f5..613ff0d6 100644 --- a/core/core.go +++ b/core/core.go @@ -734,8 +734,13 @@ func cleanIfNeeded(bazelPath string, startupOptions []string) { } } +type ParentCommit struct { + SHA string `json:"sha"` +} + type Commit struct { SHA string `json:"sha"` + PARENTS []ParentCommit `json:"parents"` } type CompareResponse struct { @@ -803,7 +808,10 @@ func getBazelCommitsBetween(goodCommit string, badCommit string) ([]string, erro } for _, commit := range compareResponse.Commits { - commitList = append(commitList, commit.SHA) + // If it has only one parent commit, add it to the list, otherwise it's a merge commit and we ignore it + if len(commit.PARENTS) == 1 { + commitList = append(commitList, commit.SHA) + } } // Check if there are more commits to fetch From 2229535e7e71669779589536cf0a842b46a7ee38 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 10 May 2023 16:04:04 +0200 Subject: [PATCH 06/22] Retry fetching Bazel versions from GCS URL (#459) * Retry fetching GCS URL * Sleep --- repositories/gcs.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/repositories/gcs.go b/repositories/gcs.go index 542e781d..db6f0b8f 100644 --- a/repositories/gcs.go +++ b/repositories/gcs.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "strings" + "time" "github.com/bazelbuild/bazelisk/core" "github.com/bazelbuild/bazelisk/httputil" @@ -77,7 +78,22 @@ func listDirectoriesInReleaseBucket(prefix string) ([]string, bool, error) { if nextPageToken != "" { url = fmt.Sprintf("%s&pageToken=%s", baseURL, nextPageToken) } - content, _, err := httputil.ReadRemoteFile(url, "") + + var content []byte + var err error + // Theoretically, this should always work, but we've seen transient + // errors on Bazel CI, so we retry a few times to work around this. + // https://github.com/bazelbuild/continuous-integration/issues/1627 + waitTime := 100 * time.Microsecond + for attempt := 0; attempt < 5; attempt++ { + content, _, err = httputil.ReadRemoteFile(url, "") + if err == nil { + break + } + time.Sleep(waitTime) + waitTime *= 2 + } + if err != nil { return nil, false, fmt.Errorf("could not list GCS objects at %s: %v", url, err) } From 63400fb16c6ba8e7cfa31dd58033ab9e83a22b51 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Fri, 12 May 2023 13:59:28 +0200 Subject: [PATCH 07/22] Improve the overriding of good bazel commit (#463) --- core/core.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/core/core.go b/core/core.go index 613ff0d6..a7a40d7a 100644 --- a/core/core.go +++ b/core/core.go @@ -745,6 +745,7 @@ type Commit struct { type CompareResponse struct { Commits []Commit `json:"commits"` + BaseCommit Commit `json:"base_commit"` MergeBaseCommit Commit `json:"merge_base_commit"` } @@ -764,7 +765,7 @@ func sendRequest(url string) (*http.Response, error) { return client.Do(req) } -func getBazelCommitsBetween(goodCommit string, badCommit string) ([]string, error) { +func getBazelCommitsBetween(goodCommit string, badCommit string) (string, []string, error) { commitList := make([]string, 0) page := 1 perPage := 250 // 250 is the maximum number of commits per page @@ -774,27 +775,27 @@ func getBazelCommitsBetween(goodCommit string, badCommit string) ([]string, erro response, err := sendRequest(url) if err != nil { - return nil, fmt.Errorf("Error fetching commit data: %v", err) + return goodCommit, nil, fmt.Errorf("Error fetching commit data: %v", err) } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { - return nil, fmt.Errorf("Error reading response body: %v", err) + return goodCommit, nil, fmt.Errorf("Error reading response body: %v", err) } if response.StatusCode == http.StatusNotFound { - return nil, fmt.Errorf("repository or commit not found: %s", string(body)) + return goodCommit, nil, fmt.Errorf("repository or commit not found: %s", string(body)) } else if response.StatusCode == 403 { - return nil, fmt.Errorf("github API rate limit hit, consider setting BAZELISK_GITHUB_TOKEN: %s", string(body)) + return goodCommit, nil, fmt.Errorf("github API rate limit hit, consider setting BAZELISK_GITHUB_TOKEN: %s", string(body)) } else if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected response status code %d: %s", response.StatusCode, string(body)) + return goodCommit, nil, fmt.Errorf("unexpected response status code %d: %s", response.StatusCode, string(body)) } var compareResponse CompareResponse err = json.Unmarshal(body, &compareResponse) if err != nil { - return nil, fmt.Errorf("Error unmarshaling JSON: %v", err) + return goodCommit, nil, fmt.Errorf("Error unmarshaling JSON: %v", err) } if len(compareResponse.Commits) == 0 { @@ -802,7 +803,7 @@ func getBazelCommitsBetween(goodCommit string, badCommit string) ([]string, erro } mergeBaseCommit := compareResponse.MergeBaseCommit.SHA - if compareResponse.MergeBaseCommit.SHA != goodCommit { + if mergeBaseCommit != compareResponse.BaseCommit.SHA { fmt.Printf("The good Bazel commit is not an ancestor of the bad Bazel commit, overriding the good Bazel commit to the merge base commit %s\n", mergeBaseCommit) goodCommit = mergeBaseCommit } @@ -823,17 +824,17 @@ func getBazelCommitsBetween(goodCommit string, badCommit string) ([]string, erro } if len(commitList) == 0 { - return nil, fmt.Errorf("no commits found between (%s, %s], the good commit should be first, maybe try with --bisect=%s..%s ?", goodCommit, badCommit, badCommit, goodCommit) + return goodCommit, nil, fmt.Errorf("no commits found between (%s, %s], the good commit should be first, maybe try with --bisect=%s..%s ?", goodCommit, badCommit, badCommit, goodCommit) } fmt.Printf("Found %d commits between (%s, %s]\n", len(commitList), goodCommit, badCommit) - return commitList, nil + return goodCommit, commitList, nil } func bisect(goodCommit string, badCommit string, args []string, bazeliskHome string, repos *Repositories) { // 1. Get the list of commits between goodCommit and badCommit fmt.Printf("\n\n--- Getting the list of commits between %s and %s\n\n", goodCommit, badCommit) - commitList, err := getBazelCommitsBetween(goodCommit, badCommit) + goodCommit, commitList, err := getBazelCommitsBetween(goodCommit, badCommit) if err != nil { log.Fatalf("Failed to get commits: %v", err) os.Exit(1) From 3629199b0fde2d63901733fc4e1ea1ed6d6f63d0 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Mon, 15 May 2023 12:43:42 +0200 Subject: [PATCH 08/22] Update doc about --bisect (#464) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 76a3010e..5e2f7e4d 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,9 @@ This will show you which flags can safely enabled, and which flags require a mig bazelisk --bisect=6.0.0..HEAD test //foo:bar_test ``` -### Useful environment variables +Note that, Bazelisk uses prebuilt Bazel binaries at commits on the main and release branches, therefore you cannot bisect your local commits. + +### Useful environment variables for --migrate and --bisect You can set `BAZELISK_INCOMPATIBLE_FLAGS` to set a list of incompatible flags (separated by `,`) to be tested, otherwise Bazelisk tests all flags starting with `--incompatible_`. From 92d73f810967a49edbe4f604a19059d661644602 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 15:12:33 +0200 Subject: [PATCH 09/22] Bump github.com/bazelbuild/rules_go from 0.38.1 to 0.39.1 (#447) Bumps [github.com/bazelbuild/rules_go](https://github.com/bazelbuild/rules_go) from 0.38.1 to 0.39.1. - [Release notes](https://github.com/bazelbuild/rules_go/releases) - [Commits](https://github.com/bazelbuild/rules_go/compare/v0.38.1...v0.39.1) --- updated-dependencies: - dependency-name: github.com/bazelbuild/rules_go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb48e45c..d22d0502 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/bazelbuild/bazelisk go 1.18 require ( - github.com/bazelbuild/rules_go v0.38.1 + github.com/bazelbuild/rules_go v0.39.1 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d github.com/hashicorp/go-version v1.6.0 github.com/mitchellh/go-homedir v1.1.0 diff --git a/go.sum b/go.sum index bf171a53..c003c621 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/bazelbuild/rules_go v0.38.1 h1:YGNsLhWe18Ielebav7cClP3GMwBxBE+xEArLHtmXDx8= -github.com/bazelbuild/rules_go v0.38.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= +github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8= +github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= From 86a4366204ba85f950e561b74e84db37c7d45f1e Mon Sep 17 00:00:00 2001 From: Julio Merino Date: Tue, 16 May 2023 06:15:46 -0700 Subject: [PATCH 10/22] Implement support for BAZELISK_FORMAT_URL (#427) This new configuration setting provides a format-like string to compute the URL from which to fetch Bazel. Takes precedence over BAZELISK_BASE_URL as this is a more general concept. Fixes #423. --- README.md | 12 +++++- bazelisk_test.sh | 23 ++++++++++-- core/BUILD | 5 ++- core/core.go | 10 ++++- core/repositories.go | 62 +++++++++++++++++++++++++++++++ core/repositories_test.go | 78 +++++++++++++++++++++++++++++++++++++++ httputil/BUILD | 6 +-- platforms/BUILD | 6 +++ platforms/platforms.go | 27 +++++++++++--- 9 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 core/repositories_test.go diff --git a/README.md b/README.md index 5e2f7e4d..6ae96fdf 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,21 @@ By default Bazelisk retrieves Bazel releases, release candidates and binaries bu As mentioned in the previous section, the `/` version format allows you to use your own Bazel fork hosted on GitHub: -If you want to create a fork with your own releases, you have to follow the naming conventions that we use in `bazelbuild/bazel` for the binary file names. +If you want to create a fork with your own releases, you should follow the naming conventions that we use in `bazelbuild/bazel` for the binary file names as this results in predictable URLs that are similar to the official ones. The URL format looks like `https://github.com//bazel/releases/download//`. You can also override the URL by setting the environment variable `$BAZELISK_BASE_URL`. Bazelisk will then append `//` to the base URL instead of using the official release server. Bazelisk will read file [`~/.netrc`](https://everything.curl.dev/usingcurl/netrc) for credentials for Basic authentication. +If for any reason none of this works, you can also override the URL format altogether by setting the environment variable `$BAZELISK_FORMAT_URL`. This variable takes a format-like string with placeholders and performs the following replacements to compute the download URL: + +- `%e`: Extension suffix, such as the empty string or `.exe`. +- `%h`: Value of `BAZELISK_VERIFY_SHA256`, respecting uppercase/lowercase characters. +- `%m`: Machine architecture name, such as `arm64` or `x86_64`. +- `%o`: Operating system name, such as `darwin` or `linux`. +- `%v`: Bazel version as determined by Bazelisk. +- `%%`: Literal `%` for escaping purposes. +- All other characters after `%` are reserved for future use and result in a processing error. + ## Ensuring that your developers use Bazelisk rather than Bazel Bazel installers typically provide Bazel's [shell wrapper script] as the `bazel` on the PATH. diff --git a/bazelisk_test.sh b/bazelisk_test.sh index 66c5b7b0..2ce2c905 100755 --- a/bazelisk_test.sh +++ b/bazelisk_test.sh @@ -197,7 +197,20 @@ function test_bazel_version_from_file() { (echo "FAIL: Expected to find 'Build label: 5.0.0' in the output of 'bazelisk version'"; exit 1) } -function test_bazel_version_from_url() { +function test_bazel_version_from_format_url() { + setup + + echo "0.19.0" > .bazelversion + + BAZELISK_FORMAT_URL="https://github.com/bazelbuild/bazel/releases/download/%v/bazel-%v-%o-%m%e" \ + BAZELISK_HOME="$BAZELISK_HOME" \ + bazelisk version 2>&1 | tee log + + grep "Build label: 0.19.0" log || \ + (echo "FAIL: Expected to find 'Build label: 0.19.0' in the output of 'bazelisk version'"; exit 1) +} + +function test_bazel_version_from_base_url() { setup echo "0.19.0" > .bazelversion @@ -427,8 +440,12 @@ if [[ $BAZELISK_VERSION == "GO" ]]; then test_bazel_last_rc echo - echo "# test_bazel_version_from_url" - test_bazel_version_from_url + echo "# test_bazel_version_from_format_url" + test_bazel_version_from_format_url + echo + + echo "# test_bazel_version_from_base_url" + test_bazel_version_from_base_url echo echo "# test_bazel_version_prefer_environment_to_bazeliskrc" diff --git a/core/BUILD b/core/BUILD index 2f5ec601..b941e3d2 100644 --- a/core/BUILD +++ b/core/BUILD @@ -19,6 +19,9 @@ go_library( go_test( name = "go_default_test", - srcs = ["core_test.go"], + srcs = [ + "core_test.go", + "repositories_test.go", + ], embed = [":go_default_library"], ) diff --git a/core/core.go b/core/core.go index a7a40d7a..ef4a66f4 100644 --- a/core/core.go +++ b/core/core.go @@ -439,8 +439,14 @@ func downloadBazelIfNecessary(version string, baseDirectory string, repos *Repos } var tmpDestPath string - if url := GetEnvOrConfig(BaseURLEnv); url != "" { - tmpDestPath, err = repos.DownloadFromBaseURL(url, version, destDir, tmpDestFile) + baseURL := GetEnvOrConfig(BaseURLEnv) + formatURL := GetEnvOrConfig(FormatURLEnv) + if baseURL != "" && formatURL != "" { + return "", fmt.Errorf("cannot set %s and %s at once", BaseURLEnv, FormatURLEnv) + } else if formatURL != "" { + tmpDestPath, err = repos.DownloadFromFormatURL(formatURL, version, destDir, tmpDestFile) + } else if baseURL != "" { + tmpDestPath, err = repos.DownloadFromBaseURL(baseURL, version, destDir, tmpDestFile) } else { tmpDestPath, err = downloader(destDir, tmpDestFile) } diff --git a/core/repositories.go b/core/repositories.go index f0ab7327..7b5b4be6 100644 --- a/core/repositories.go +++ b/core/repositories.go @@ -13,6 +13,9 @@ import ( const ( // BaseURLEnv is the name of the environment variable that stores the base URL for downloads. BaseURLEnv = "BAZELISK_BASE_URL" + + // FormatURLEnv is the name of the environment variable that stores the format string to generate URLs for downloads. + FormatURLEnv = "BAZELISK_FORMAT_URL" ) // DownloadFunc downloads a specific Bazel binary to the given location and returns the absolute path. @@ -232,6 +235,65 @@ func (r *Repositories) DownloadFromBaseURL(baseURL, version, destDir, destFile s return httputil.DownloadBinary(url, destDir, destFile) } +func BuildURLFromFormat(formatURL, version string) (string, error) { + osName, err := platforms.DetermineOperatingSystem() + if err != nil { + return "", err + } + + machineName, err := platforms.DetermineArchitecture(osName, version) + if err != nil { + return "", err + } + + var b strings.Builder + b.Grow(len(formatURL) * 2) // Approximation. + for i := 0; i < len(formatURL); i++ { + ch := formatURL[i] + if ch == '%' { + i++ + if i == len(formatURL) { + return "", errors.New("trailing %") + } + + ch = formatURL[i] + switch ch { + case 'e': + b.WriteString(platforms.DetermineExecutableFilenameSuffix()) + case 'h': + b.WriteString(GetEnvOrConfig("BAZELISK_VERIFY_SHA256")) + case 'm': + b.WriteString(machineName) + case 'o': + b.WriteString(osName) + case 'v': + b.WriteString(version) + case '%': + b.WriteByte('%') + default: + return "", fmt.Errorf("unknown placeholder %%%c", ch) + } + } else { + b.WriteByte(ch) + } + } + return b.String(), nil +} + +// DownloadFromFormatURL can download Bazel binaries from a specific URL while ignoring the predefined repositories. +func (r *Repositories) DownloadFromFormatURL(formatURL, version, destDir, destFile string) (string, error) { + if formatURL == "" { + return "", fmt.Errorf("%s is not set", FormatURLEnv) + } + + url, err := BuildURLFromFormat(formatURL, version) + if err != nil { + return "", err + } + + return httputil.DownloadBinary(url, destDir, destFile) +} + // CreateRepositories creates a new Repositories instance with the given repositories. Any nil repository will be replaced by a dummy repository that raises an error whenever a download is attempted. func CreateRepositories(releases ReleaseRepo, candidates CandidateRepo, fork ForkRepo, commits CommitRepo, rolling RollingRepo, supportsBaseURL bool) *Repositories { repos := &Repositories{supportsBaseURL: supportsBaseURL} diff --git a/core/repositories_test.go b/core/repositories_test.go new file mode 100644 index 00000000..f6442446 --- /dev/null +++ b/core/repositories_test.go @@ -0,0 +1,78 @@ +package core + +import ( + "errors" + "fmt" + "os" + "testing" + + "github.com/bazelbuild/bazelisk/platforms" +) + +func TestBuildURLFromFormat(t *testing.T) { + osName, err := platforms.DetermineOperatingSystem() + if err != nil { + t.Fatalf("Cannot get operating system name: %v", err) + } + + version := "6.0.0" + + machineName, err := platforms.DetermineArchitecture(osName, version) + if err != nil { + t.Fatalf("Cannot get machine architecture name: %v", err) + } + + suffix := platforms.DetermineExecutableFilenameSuffix() + + previousSha256, hadSha256 := os.LookupEnv("BAZELISK_VERIFY_SHA256") + sha256 := "SomeSha256ValueThatIsIrrelevant" + if err := os.Setenv("BAZELISK_VERIFY_SHA256", sha256); err != nil { + t.Fatalf("Failed to set BAZELISK_VERIFY_SHA256") + } + defer func() { + if hadSha256 { + os.Setenv("BAZELISK_VERIFY_SHA256", previousSha256) + } else { + os.Unsetenv("BAZELISK_VERIFY_SHA256") + } + }() + + type test struct { + format string + want string + wantErr error + } + + tests := []test{ + {format: "", want: ""}, + {format: "no/placeholders", want: "no/placeholders"}, + + {format: "%", wantErr: errors.New("trailing %")}, + {format: "%%", want: "%"}, + {format: "%%%%", want: "%%"}, + {format: "invalid/trailing/%", wantErr: errors.New("trailing %")}, + {format: "escaped%%placeholder", want: "escaped%placeholder"}, + + {format: "foo-%e-bar", want: fmt.Sprintf("foo-%s-bar", suffix)}, + {format: "foo-%h-bar", want: fmt.Sprintf("foo-%s-bar", sha256)}, + {format: "foo-%m-bar", want: fmt.Sprintf("foo-%s-bar", machineName)}, + {format: "foo-%o-bar", want: fmt.Sprintf("foo-%s-bar", osName)}, + {format: "foo-%v-bar", want: fmt.Sprintf("foo-%s-bar", version)}, + + {format: "repeated %v %m %v", want: fmt.Sprintf("repeated %s %s %s", version, machineName, version)}, + + {format: "https://real.example.com/%e/%m/%o/%v#%%20trailing", want: fmt.Sprintf("https://real.example.com/%s/%s/%s/%s#%%20trailing", suffix, machineName, osName, version)}, + } + + for _, tc := range tests { + got, err := BuildURLFromFormat(tc.format, version) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tc.wantErr) { + if got != "" { + t.Errorf("format '%s': got non-empty '%s' on error", tc.format, got) + } + t.Errorf("format '%s': got error %v, want error %v", tc.format, err, tc.wantErr) + } else if got != tc.want { + t.Errorf("format '%s': got %s, want %s", tc.format, got, tc.want) + } + } +} diff --git a/httputil/BUILD b/httputil/BUILD index f168cffa..d48f2902 100644 --- a/httputil/BUILD +++ b/httputil/BUILD @@ -10,12 +10,12 @@ go_library( "fake.go", "httputil.go", ], + importpath = "github.com/bazelbuild/bazelisk/httputil", + visibility = ["//visibility:public"], deps = [ + "@com_github_bgentry_go_netrc//:go_default_library", "@com_github_mitchellh_go_homedir//:go_default_library", - "@com_github_bgentry_go_netrc//:go_default_library" ], - importpath = "github.com/bazelbuild/bazelisk/httputil", - visibility = ["//visibility:public"], ) go_test( diff --git a/platforms/BUILD b/platforms/BUILD index dcb655f5..8ab333f9 100644 --- a/platforms/BUILD +++ b/platforms/BUILD @@ -16,3 +16,9 @@ go_test( srcs = ["platforms_test.go"], embed = [":go_default_library"], ) + +go_test( + name = "go_default_test", + srcs = ["platforms_test.go"], + embed = [":go_default_library"], +) diff --git a/platforms/platforms.go b/platforms/platforms.go index 3c0fcbe0..2e918dcd 100644 --- a/platforms/platforms.go +++ b/platforms/platforms.go @@ -54,8 +54,7 @@ func DetermineExecutableFilenameSuffix() string { return filenameSuffix } -// DetermineBazelFilename returns the correct file name of a local Bazel binary. -func DetermineBazelFilename(version string, includeSuffix bool) (string, error) { +func DetermineArchitecture(osName, version string) (string, error) { var machineName string switch runtime.GOARCH { case "amd64": @@ -66,16 +65,32 @@ func DetermineBazelFilename(version string, includeSuffix bool) (string, error) return "", fmt.Errorf("unsupported machine architecture \"%s\", must be arm64 or x86_64", runtime.GOARCH) } - var osName string + if osName == "darwin" { + machineName = DarwinFallback(machineName, version) + } + + return machineName, nil +} + +func DetermineOperatingSystem() (string, error) { switch runtime.GOOS { case "darwin", "linux", "windows": - osName = runtime.GOOS + return runtime.GOOS, nil default: return "", fmt.Errorf("unsupported operating system \"%s\", must be Linux, macOS or Windows", runtime.GOOS) } +} - if osName == "darwin" { - machineName = DarwinFallback(machineName, version) +// DetermineBazelFilename returns the correct file name of a local Bazel binary. +func DetermineBazelFilename(version string, includeSuffix bool) (string, error) { + osName, err := DetermineOperatingSystem() + if err != nil { + return "", err + } + + machineName, err := DetermineArchitecture(osName, version) + if err != nil { + return "", err } var filenameSuffix string From 70e3e87d4ca23cdbe5439685fb6d2018d69be1e5 Mon Sep 17 00:00:00 2001 From: Jeremy Nimmer Date: Tue, 16 May 2023 06:19:18 -0700 Subject: [PATCH 11/22] bazelisk.py notices WORKSPACE.bazel as a workspace_root (#443) --- bazelisk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bazelisk.py b/bazelisk.py index 44f4256f..8a967c27 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -97,6 +97,8 @@ def find_workspace_root(root=None): root = os.getcwd() if os.path.exists(os.path.join(root, "WORKSPACE")): return root + if os.path.exists(os.path.join(root, "WORKSPACE.bazel")): + return root new_root = os.path.dirname(root) return find_workspace_root(new_root) if new_root != root else None From 207d51b5ac824a4c2b60bb861d0827fcf61f2137 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 19 May 2023 20:51:04 +0200 Subject: [PATCH 12/22] Remove duplicated test (#467) Duplicated `go_tests` can result in compilation actions failing due to https://github.com/bazelbuild/rules_go/issues/3558. --- platforms/BUILD | 6 ------ 1 file changed, 6 deletions(-) diff --git a/platforms/BUILD b/platforms/BUILD index 8ab333f9..cb6950d8 100644 --- a/platforms/BUILD +++ b/platforms/BUILD @@ -11,12 +11,6 @@ go_library( ], ) -go_test( - name = "platforms_test", - srcs = ["platforms_test.go"], - embed = [":go_default_library"], -) - go_test( name = "go_default_test", srcs = ["platforms_test.go"], From 748a6da6f20eaa98dc87c58eb56eab8e7099572e Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 19 May 2023 20:55:18 +0200 Subject: [PATCH 13/22] Remove unused `runfiles` package (#468) --- runfiles/BUILD | 11 ----------- runfiles/runfiles.go | 20 -------------------- 2 files changed, 31 deletions(-) delete mode 100644 runfiles/BUILD delete mode 100644 runfiles/runfiles.go diff --git a/runfiles/BUILD b/runfiles/BUILD deleted file mode 100644 index 46e4411c..00000000 --- a/runfiles/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["runfiles.go"], - importpath = "github.com/bazelbuild/bazelisk/runfiles", - visibility = ["//visibility:public"], - deps = [ - "@io_bazel_rules_go//go/tools/bazel:go_default_library", - ], -) diff --git a/runfiles/runfiles.go b/runfiles/runfiles.go deleted file mode 100644 index 82c204cf..00000000 --- a/runfiles/runfiles.go +++ /dev/null @@ -1,20 +0,0 @@ -// Package runfiles offers functionality to read data dependencies of tests. -package runfiles - -import ( - "io/ioutil" - - "github.com/bazelbuild/rules_go/go/tools/bazel" -) - -func ReadFile(name string) ([]byte, error) { - path, err := bazel.Runfile(name) - if err != nil { - return nil, err - } - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return data, nil -} From ad2f27d8ed7bc3774d97bc40cc2834265b75f85d Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Fri, 26 May 2023 12:22:33 +0200 Subject: [PATCH 14/22] More examples for `bazelisk --bisect` (#470) --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 6ae96fdf..35d46226 100644 --- a/README.md +++ b/README.md @@ -131,8 +131,16 @@ This will show you which flags can safely enabled, and which flags require a mig `--bisect` flag allows you to bisect Bazel versions to find which version introduced a build failure. You can specify the range of versions to bisect with `--bisect=..`, where GOOD is the last known working Bazel version and BAD is the first known non-working Bazel version. Bazelisk uses [GitHub's compare API](https://docs.github.com/en/rest/commits/commits#compare-two-commits) to get the list of commits to bisect. When GOOD is not an ancestor of BAD, GOOD is reset to their merge base commit. +Examples: ```shell +# Bisect between 6.0.0 and Bazel at HEAD bazelisk --bisect=6.0.0..HEAD test //foo:bar_test + +# Bisect between 6.1.0 and the second release candidate of Bazel 6.2.0 +bazelisk --bisect=6.1.0..release-6.2.0rc2 test //foo:bar_test + +# Bisect between two commits on the main branch (or branches with `release-` prefix) of the Bazel GitHub repository. +bazelisk --bisect=.. test //foo:bar_test ``` Note that, Bazelisk uses prebuilt Bazel binaries at commits on the main and release branches, therefore you cannot bisect your local commits. From a4dca9c07203ab550bd19a4dbbc3cbc545e63c42 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Jun 2023 17:19:22 +0100 Subject: [PATCH 15/22] Use consistent paths to bazel binaries (#465) Before this change, changing which mirror you were downloading from would change the $PATH environment variable bazel is run with, even though the bazel binaries being pointed to are identical. This can cause repository rules to invalidate and re-run. Instead, store downloaded bazels in directories keyed off of the sha256 of the bazel binary itself, and track the metadata of a mirror+version -> sha256. This avoid spurious rebuilds when only changing the URL bazelisk would use to download bazel. --- bazelisk_test.sh | 78 +++++++++++++++++++++++++++-- core/core.go | 125 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 161 insertions(+), 42 deletions(-) diff --git a/bazelisk_test.sh b/bazelisk_test.sh index 2ce2c905..95b51d69 100755 --- a/bazelisk_test.sh +++ b/bazelisk_test.sh @@ -299,6 +299,52 @@ EOF (echo "FAIL: Expected to find 'BAZELISK_SKIP_WRAPPER=true' in the output of 'bazelisk version'"; exit 1) } +function test_path_is_consistent_regardless_of_base_url() { + setup + + echo 6.2.0 > .bazelversion + + cat >WORKSPACE <print_path.bzl <&1 | tee log1 + + BAZELISK_HOME="$BAZELISK_HOME" bazelisk clean --expunge 2>&1 + + # We need a separate mirror of bazel binaries, which has identical files. + # Ideally we wouldn't depend on sourceforge for test runtime, but hey, it exists and it works. + BAZELISK_HOME="$BAZELISK_HOME" BAZELISK_BASE_URL=https://downloads.sourceforge.net/project/bazel.mirror bazelisk sync --only=print_path 2>&1 | tee log2 + + path1="$(grep "PATH is:" log1)" + path2="$(grep "PATH is:" log2)" + + [[ -n "${path1}" && -n "${path2}" ]] || \ + (echo "FAIL: Expected PATH to be non-empty, got path1=${path1}, path2=${path2}"; exit 1) + + [[ "${path1}" == "${path2}" ]] || \ + (echo "FAIL: Expected PATH to be the same regardless of which mirror was used, got path1=${path1}, path2=${path2}"; exit 1) +} + function test_skip_wrapper() { setup @@ -327,10 +373,10 @@ function test_bazel_download_path_go() { BAZELISK_HOME="$BAZELISK_HOME" \ bazelisk version 2>&1 | tee log - find "$BAZELISK_HOME/downloads/bazelbuild" 2>&1 | tee log + find "$BAZELISK_HOME/downloads/metadata/bazelbuild" 2>&1 | tee log - grep "^$BAZELISK_HOME/downloads/bazelbuild/bazel-[0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*-[a-z0-9_-]*/bin/bazel\(.exe\)\?$" log || \ - (echo "FAIL: Expected to download bazel binary into specific path."; exit 1) + grep "^$BAZELISK_HOME/downloads/metadata/bazelbuild/bazel-[0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*-[a-z0-9_-]*$" log || \ + (echo "FAIL: Expected to download bazel metadata in specific path."; exit 1) } function test_bazel_verify_sha256() { @@ -395,8 +441,26 @@ function test_bazel_prepend_binary_directory_to_path_go() { BAZELISK_HOME="$BAZELISK_HOME" \ bazelisk --print_env 2>&1 | tee log - PATTERN=$(echo "^PATH=$BAZELISK_HOME/downloads/bazelbuild/bazel-[0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*-[a-z0-9_-]*/bin[:;]" | sed -e 's/\//\[\/\\\\\]/g') - grep "$PATTERN" log || \ + local os="$(uname -s | tr A-Z a-z)" + case "${os}" in + darwin|linux) + path_entry_delimiter=":" + path_delimiter="/" + extension="" + ;; + msys*|mingw*|cygwin*) + path_entry_delimiter=";" + path_delimiter="\\" + extension=".exe" + ;; + *) + echo "FAIL: Unknown OS ${os} in test" + exit 1 + ;; + esac + path_entry="$(grep "^PATH=" log | cut -d= -f2- | cut -d"${path_entry_delimiter}" -f1)" + + [[ -x "${path_entry}${path_delimiter}bazel${extension}" ]] || \ (echo "FAIL: Expected PATH to contains bazel binary directory."; exit 1) } @@ -480,6 +544,10 @@ if [[ $BAZELISK_VERSION == "GO" ]]; then test_bazel_prepend_binary_directory_to_path_go echo + echo "# test_path_is_consistent_regardless_of_base_url" + test_path_is_consistent_regardless_of_base_url + echo + case "$(uname -s)" in MSYS*) # The tests are currently not compatible with Windows. diff --git a/core/core.go b/core/core.go index ef4a66f4..f08a3e66 100644 --- a/core/core.go +++ b/core/core.go @@ -5,6 +5,7 @@ package core import ( "bufio" + "crypto/rand" "crypto/sha256" "encoding/json" "fmt" @@ -416,73 +417,123 @@ func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Reposi bazelForkOrURL = bazelFork } - baseDirectory := filepath.Join(bazeliskHome, "downloads", bazelForkOrURL) - bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, baseDirectory, repos, downloader) + bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, bazeliskHome, bazelForkOrURL, repos, downloader) return bazelPath, err } -func downloadBazelIfNecessary(version string, baseDirectory string, repos *Repositories, downloader DownloadFunc) (string, error) { +// downloadBazelIfNecessary returns a path to a bazel which can be run, which may have been cached. +// The directory it returns may depend on version and bazeliskHome, but does not depend on bazelForkOrURLDirName. +// This is important, as the directory may be added to $PATH, and varying the path for equivalent files may cause unnecessary repository rule cache invalidations. +// Where a file was downloaded from shouldn't affect cache behaviour of Bazel invocations. +// +// The structure of the downloads directory is as follows ([]s indicate variables): +// +// downloads/metadata/[fork-or-url]/bazel-[version-os-etc] is a text file containing a hex sha256 of the contents of the downloaded bazel file. +// downloads/sha256/[sha256]/bin/bazel[extension] contains the bazel with a particular sha256. +func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrURLDirName string, repos *Repositories, downloader DownloadFunc) (string, error) { pathSegment, err := platforms.DetermineBazelFilename(version, false) if err != nil { return "", fmt.Errorf("could not determine path segment to use for Bazel binary: %v", err) } + destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() + + mappingPath := filepath.Join(bazeliskHome, "downloads", "metadata", bazelForkOrURLDirName, pathSegment) + digestFromMappingFile, err := os.ReadFile(mappingPath) + if err == nil { + pathToBazelInCAS := filepath.Join(bazeliskHome, "downloads", "sha256", string(digestFromMappingFile), "bin", destFile) + if _, err := os.Stat(pathToBazelInCAS); err == nil { + return pathToBazelInCAS, nil + } + } + + pathToBazelInCAS, downloadedDigest, err := downloadBazelToCAS(version, bazeliskHome, repos, downloader) + if err != nil { + return "", fmt.Errorf("failed to download bazel: %w", err) + } - destDir := filepath.Join(baseDirectory, pathSegment, "bin") expectedSha256 := strings.ToLower(GetEnvOrConfig("BAZELISK_VERIFY_SHA256")) + if len(expectedSha256) > 0 { + if expectedSha256 != downloadedDigest { + return "", fmt.Errorf("%s has sha256=%s but need sha256=%s", pathToBazelInCAS, downloadedDigest, expectedSha256) + } + } - tmpDestFile := "bazel-tmp" + platforms.DetermineExecutableFilenameSuffix() - destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() + if err := atomicWriteFile(mappingPath, []byte(downloadedDigest), 0644); err != nil { + return "", fmt.Errorf("failed to write mapping file after downloading bazel: %w", err) + } - destPath := filepath.Join(destDir, destFile) - if _, err := os.Stat(destPath); err == nil { - return destPath, nil + return pathToBazelInCAS, nil +} + +func atomicWriteFile(path string, contents []byte, perm os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("failed to MkdirAll parent of %s: %w", path, err) + } + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, contents, perm); err != nil { + return fmt.Errorf("failed to write file %s: %w", tmpPath, err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("failed to rename %s to %s: %w", tmpPath, path, err) } + return nil +} + +func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories, downloader DownloadFunc) (string, string, error) { + downloadsDir := filepath.Join(bazeliskHome, "downloads") + temporaryDownloadDir := filepath.Join(downloadsDir, "_tmp") + casDir := filepath.Join(bazeliskHome, "downloads", "sha256") + + tmpDestFileBytes := make([]byte, 32) + if _, err := rand.Read(tmpDestFileBytes); err != nil { + return "", "", fmt.Errorf("failed to generate temporary file name: %w", err) + } + tmpDestFile := fmt.Sprintf("%x", tmpDestFileBytes) var tmpDestPath string + var err error baseURL := GetEnvOrConfig(BaseURLEnv) formatURL := GetEnvOrConfig(FormatURLEnv) if baseURL != "" && formatURL != "" { - return "", fmt.Errorf("cannot set %s and %s at once", BaseURLEnv, FormatURLEnv) + return "", "", fmt.Errorf("cannot set %s and %s at once", BaseURLEnv, FormatURLEnv) } else if formatURL != "" { - tmpDestPath, err = repos.DownloadFromFormatURL(formatURL, version, destDir, tmpDestFile) + tmpDestPath, err = repos.DownloadFromFormatURL(formatURL, version, temporaryDownloadDir, tmpDestFile) } else if baseURL != "" { - tmpDestPath, err = repos.DownloadFromBaseURL(baseURL, version, destDir, tmpDestFile) + tmpDestPath, err = repos.DownloadFromBaseURL(baseURL, version, temporaryDownloadDir, tmpDestFile) } else { - tmpDestPath, err = downloader(destDir, tmpDestFile) + tmpDestPath, err = downloader(temporaryDownloadDir, tmpDestFile) } if err != nil { - return "", err + return "", "", fmt.Errorf("failed to download bazel: %w", err) } - if len(expectedSha256) > 0 { - f, err := os.Open(tmpDestPath) - if err != nil { - os.Remove(tmpDestPath) - return "", fmt.Errorf("cannot open %s after download: %v", tmpDestPath, err) - } - defer os.Remove(tmpDestPath) - // We cannot defer f.Close() because keeping the handle open when we try to do the - // rename later on fails on Windows. + f, err := os.Open(tmpDestPath) + if err != nil { + return "", "", fmt.Errorf("failed to open downloaded bazel to digest it: %w", err) + } - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - f.Close() - return "", fmt.Errorf("cannot compute sha256 of %s after download: %v", tmpDestPath, err) - } + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { f.Close() + return "", "", fmt.Errorf("cannot compute sha256 of %s after download: %v", tmpDestPath, err) + } + f.Close() + actualSha256 := strings.ToLower(fmt.Sprintf("%x", h.Sum(nil))) - actualSha256 := strings.ToLower(fmt.Sprintf("%x", h.Sum(nil))) - if expectedSha256 != actualSha256 { - return "", fmt.Errorf("%s has sha256=%s but need sha256=%s", tmpDestPath, actualSha256, expectedSha256) - } + pathToBazelInCAS := filepath.Join(casDir, actualSha256, "bin", "bazel"+platforms.DetermineExecutableFilenameSuffix()) + if err := os.MkdirAll(filepath.Dir(pathToBazelInCAS), 0755); err != nil { + return "", "", fmt.Errorf("failed to MkdirAll parent of %s: %w", pathToBazelInCAS, err) } - // Only place the downloaded binary in its final location once we know it is fully downloaded - // and valid, to prevent invalid files from ever being executed. - if err = os.Rename(tmpDestPath, destPath); err != nil { - return "", fmt.Errorf("cannot rename %s to %s: %v", tmpDestPath, destPath, err) + tmpPathInCorrectDirectory := pathToBazelInCAS + ".tmp" + if err := os.Rename(tmpDestPath, tmpPathInCorrectDirectory); err != nil { + return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpDestPath, tmpPathInCorrectDirectory, err) + } + if err := os.Rename(tmpPathInCorrectDirectory, pathToBazelInCAS); err != nil { + return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpPathInCorrectDirectory, pathToBazelInCAS, err) } - return destPath, nil + + return pathToBazelInCAS, actualSha256, nil } func copyFile(src, dst string, perm os.FileMode) error { From d0e93d2b84d5d092f5bc116a00bee0e368e1a771 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Jun 2023 17:38:34 +0100 Subject: [PATCH 16/22] Extract config fetching into its own package (#462) My goal here is to be able to pass my own config implementation when calling RunBazelisk from code without setting environment variables, because they cause repository rule cache invalidations. As a side-effect, it helps towards the TODO in core.go to split functionality into packages. --- bazelisk.go | 5 +- config/BUILD | 12 ++ config/config.go | 130 +++++++++++++++ core/BUILD | 5 + core/core.go | 336 ++++++++++++++------------------------ core/core_test.go | 38 +++-- core/repositories.go | 9 +- core/repositories_test.go | 18 +- ws/BUILD | 11 ++ ws/ws.go | 36 ++++ 10 files changed, 353 insertions(+), 247 deletions(-) create mode 100644 config/BUILD create mode 100644 config/config.go create mode 100644 ws/BUILD create mode 100644 ws/ws.go diff --git a/bazelisk.go b/bazelisk.go index 5cdee070..a25041c1 100644 --- a/bazelisk.go +++ b/bazelisk.go @@ -25,11 +25,12 @@ import ( func main() { gcs := &repositories.GCSRepo{} - gitHub := repositories.CreateGitHubRepo(core.GetEnvOrConfig("BAZELISK_GITHUB_TOKEN")) + config := core.MakeDefaultConfig() + gitHub := repositories.CreateGitHubRepo(config.Get("BAZELISK_GITHUB_TOKEN")) // Fetch LTS releases, release candidates, rolling releases and Bazel-at-commits from GCS, forks from GitHub. repos := core.CreateRepositories(gcs, gcs, gitHub, gcs, gcs, true) - exitCode, err := core.RunBazelisk(os.Args[1:], repos) + exitCode, err := core.RunBazeliskWithArgsFuncAndConfig(func(string) []string { return os.Args[1:] }, repos, config) if err != nil { log.Fatal(err) } diff --git a/config/BUILD b/config/BUILD new file mode 100644 index 00000000..4752410e --- /dev/null +++ b/config/BUILD @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["config.go"], + importpath = "github.com/bazelbuild/bazelisk/config", + visibility = ["//visibility:public"], + deps = [ + "//ws:go_default_library", + "@com_github_mitchellh_go_homedir//:go_default_library", + ], +) diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..25afbe4e --- /dev/null +++ b/config/config.go @@ -0,0 +1,130 @@ +package config + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/bazelbuild/bazelisk/ws" +) + +const rcFileName = ".bazeliskrc" + +// Config allows getting Bazelisk configuration values. +type Config interface { + Get(name string) string +} + +// FromEnv returns a Config which gets config values from environment variables. +func FromEnv() Config { + return &fromEnv{} +} + +type fromEnv struct{} + +func (c *fromEnv) Get(name string) string { + return os.Getenv(name) +} + +// FromFile returns a Config which gets config values from a Bazelisk config file. +func FromFile(path string) (Config, error) { + values, err := parseFileConfig(path) + if err != nil { + return nil, err + } + return &static{ + values: values, + }, nil +} + +type static struct { + values map[string]string +} + +func (c *static) Get(name string) string { + return c.values[name] +} + +// parseFileConfig parses a .bazeliskrc file as a map of key-value configuration values. +func parseFileConfig(rcFilePath string) (map[string]string, error) { + config := make(map[string]string) + + contents, err := ioutil.ReadFile(rcFilePath) + if err != nil { + if os.IsNotExist(err) { + // Non-critical error. + return config, nil + } + return nil, err + } + + for _, line := range strings.Split(string(contents), "\n") { + if strings.HasPrefix(line, "#") { + // comments + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) < 2 { + continue + } + key := strings.TrimSpace(parts[0]) + config[key] = strings.TrimSpace(parts[1]) + } + + return config, nil +} + +// LocateUserConfigFile locates a .bazeliskrc file in the user's home directory. +func LocateUserConfigFile() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, rcFileName), nil +} + +// LocateWorkspaceConfigFile locates a .bazeliskrc file in the current workspace root. +func LocateWorkspaceConfigFile() (string, error) { + workingDirectory, err := os.Getwd() + if err != nil { + return "", err + } + workspaceRoot := ws.FindWorkspaceRoot(workingDirectory) + if workspaceRoot == "" { + return "", err + } + return filepath.Join(workspaceRoot, rcFileName), nil +} + +// Layered returns a Config which gets config values from the first of a series of other Config values which sets the config. +func Layered(configs ...Config) Config { + return &layered{ + configs: configs, + } +} + +type layered struct { + configs []Config +} + +func (c *layered) Get(name string) string { + for _, config := range c.configs { + if value := config.Get(name); value != "" { + return value + } + } + return "" +} + +// Null returns a Config with no config values. +func Null() Config { + return &static{} +} + +// Static returns a Config with static values. +func Static(values map[string]string) Config { + return &static{ + values: values, + } +} diff --git a/core/BUILD b/core/BUILD index b941e3d2..14585169 100644 --- a/core/BUILD +++ b/core/BUILD @@ -10,9 +10,11 @@ go_library( visibility = ["//visibility:public"], x_defs = {"BazeliskVersion": "{STABLE_VERSION}"}, deps = [ + "//config:go_default_library", "//httputil:go_default_library", "//platforms:go_default_library", "//versions:go_default_library", + "//ws:go_default_library", "@com_github_mitchellh_go_homedir//:go_default_library", ], ) @@ -24,4 +26,7 @@ go_test( "repositories_test.go", ], embed = [":go_default_library"], + deps = [ + "//config:go_default_library", + ], ) diff --git a/core/core.go b/core/core.go index f08a3e66..cab98d9b 100644 --- a/core/core.go +++ b/core/core.go @@ -24,9 +24,11 @@ import ( "sync" "syscall" + "github.com/bazelbuild/bazelisk/config" "github.com/bazelbuild/bazelisk/httputil" "github.com/bazelbuild/bazelisk/platforms" "github.com/bazelbuild/bazelisk/versions" + "github.com/bazelbuild/bazelisk/ws" "github.com/mitchellh/go-homedir" ) @@ -34,7 +36,6 @@ const ( bazelReal = "BAZEL_REAL" skipWrapperEnv = "BAZELISK_SKIP_WRAPPER" wrapperPath = "./tools/bazel" - rcFileName = ".bazeliskrc" maxDirLength = 255 ) @@ -50,6 +51,29 @@ var ( // Bazel with. type ArgsFunc func(resolvedBazelVersion string) []string +func MakeDefaultConfig() config.Config { + configs := []config.Config{config.FromEnv()} + + workspaceConfigPath, err := config.LocateWorkspaceConfigFile() + if err == nil { + c, err := config.FromFile(workspaceConfigPath) + if err != nil { + log.Fatal(err) + } + configs = append(configs, c) + } + + userConfigPath, err := config.LocateUserConfigFile() + if err == nil { + c, err := config.FromFile(userConfigPath) + if err != nil { + log.Fatal(err) + } + configs = append(configs, c) + } + return config.Layered(configs...) +} + // RunBazelisk runs the main Bazelisk logic for the given arguments and Bazel repositories. func RunBazelisk(args []string, repos *Repositories) (int, error) { return RunBazeliskWithArgsFunc(func(_ string) []string { return args }, repos) @@ -58,9 +82,16 @@ func RunBazelisk(args []string, repos *Repositories) (int, error) { // RunBazeliskWithArgsFunc runs the main Bazelisk logic for the given ArgsFunc and Bazel // repositories. func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error) { - httputil.UserAgent = getUserAgent() - bazeliskHome := GetEnvOrConfig("BAZELISK_HOME") + return RunBazeliskWithArgsFuncAndConfig(argsFunc, repos, MakeDefaultConfig()) +} + +// RunBazeliskWithArgsFuncAndConfig runs the main Bazelisk logic for the given ArgsFunc and Bazel +// repositories and config. +func RunBazeliskWithArgsFuncAndConfig(argsFunc ArgsFunc, repos *Repositories, config config.Config) (int, error) { + httputil.UserAgent = getUserAgent(config) + + bazeliskHome := config.Get("BAZELISK_HOME") if len(bazeliskHome) == 0 { userCacheDir, err := os.UserCacheDir() if err != nil { @@ -75,7 +106,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error return -1, fmt.Errorf("could not create directory %s: %v", bazeliskHome, err) } - bazelVersionString, err := getBazelVersion() + bazelVersionString, err := getBazelVersion(config) if err != nil { return -1, fmt.Errorf("could not get Bazel version: %v", err) } @@ -92,7 +123,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error // If we aren't using a local Bazel binary, we'll have to parse the version string and // download the version that the user wants. if !filepath.IsAbs(bazelPath) { - bazelPath, err = downloadBazel(bazelVersionString, bazeliskHome, repos) + bazelPath, err = downloadBazel(bazelVersionString, bazeliskHome, repos, config) if err != nil { return -1, fmt.Errorf("could not download Bazel: %v", err) } @@ -109,7 +140,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error // --print_env must be the first argument. if len(args) > 0 && args[0] == "--print_env" { // print environment variables for sub-processes - cmd := makeBazelCmd(bazelPath, args, nil) + cmd := makeBazelCmd(bazelPath, args, nil, config) for _, val := range cmd.Env { fmt.Println(val) } @@ -122,12 +153,12 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error if err != nil { return -1, err } - newFlags, err := getIncompatibleFlags(bazelPath, cmd) + newFlags, err := getIncompatibleFlags(bazelPath, cmd, config) if err != nil { return -1, fmt.Errorf("could not get the list of incompatible flags: %v", err) } if args[0] == "--migrate" { - migrate(bazelPath, args[1:], newFlags) + migrate(bazelPath, args[1:], newFlags, config) } else { // When --strict is present, it expands to the list of --incompatible_ flags // that should be enabled for the given Bazel version. @@ -141,7 +172,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error value := args[0][len("--bisect="):] commits := strings.Split(value, "..") if len(commits) == 2 { - bisect(commits[0], commits[1], args[1:], bazeliskHome, repos) + bisect(commits[0], commits[1], args[1:], bazeliskHome, repos, config) } else { return -1, fmt.Errorf("Error: Invalid format for --bisect. Expected format: '--bisect=..'") } @@ -167,7 +198,7 @@ func RunBazeliskWithArgsFunc(argsFunc ArgsFunc, repos *Repositories) (int, error } } - exitCode, err := runBazel(bazelPath, args, nil) + exitCode, err := runBazel(bazelPath, args, nil, config) if err != nil { return -1, fmt.Errorf("could not run Bazel: %v", err) } @@ -183,130 +214,14 @@ func getBazelCommand(args []string) (string, error) { return "", fmt.Errorf("could not find a valid Bazel command in %q. Please run `bazel help` if you need help on how to use Bazel", strings.Join(args, " ")) } -func getUserAgent() string { - agent := GetEnvOrConfig("BAZELISK_USER_AGENT") +func getUserAgent(config config.Config) string { + agent := config.Get("BAZELISK_USER_AGENT") if len(agent) > 0 { return agent } return fmt.Sprintf("Bazelisk/%s", BazeliskVersion) } -// GetEnvOrConfig reads a configuration value from the environment, but fall back to reading it from .bazeliskrc in the workspace root. -func GetEnvOrConfig(name string) string { - if val := os.Getenv(name); val != "" { - return val - } - - fileConfigOnce.Do(loadFileConfig) - - return fileConfig[name] -} - -// loadFileConfig locates available .bazeliskrc configuration files, parses them with a precedence order preference, -// and updates a global configuration map with their contents. This routine should be executed exactly once. -func loadFileConfig() { - var rcFilePaths []string - - if userRC, err := locateUserConfigFile(); err == nil { - rcFilePaths = append(rcFilePaths, userRC) - } - if workspaceRC, err := locateWorkspaceConfigFile(); err == nil { - rcFilePaths = append(rcFilePaths, workspaceRC) - } - - fileConfig = make(map[string]string) - for _, rcPath := range rcFilePaths { - config, err := parseFileConfig(rcPath) - if err != nil { - log.Fatal(err) - } - - for key, value := range config { - fileConfig[key] = value - } - } -} - -// locateWorkspaceConfigFile locates a .bazeliskrc file in the current workspace root. -func locateWorkspaceConfigFile() (string, error) { - workingDirectory, err := os.Getwd() - if err != nil { - return "", err - } - workspaceRoot := findWorkspaceRoot(workingDirectory) - if workspaceRoot == "" { - return "", err - } - return filepath.Join(workspaceRoot, rcFileName), nil -} - -// locateUserConfigFile locates a .bazeliskrc file in the user's home directory. -func locateUserConfigFile() (string, error) { - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, rcFileName), nil -} - -// parseFileConfig parses a .bazeliskrc file as a map of key-value configuration values. -func parseFileConfig(rcFilePath string) (map[string]string, error) { - config := make(map[string]string) - - contents, err := ioutil.ReadFile(rcFilePath) - if err != nil { - if os.IsNotExist(err) { - // Non-critical error. - return config, nil - } - return nil, err - } - - for _, line := range strings.Split(string(contents), "\n") { - if strings.HasPrefix(line, "#") { - // comments - continue - } - parts := strings.SplitN(line, "=", 2) - if len(parts) < 2 { - continue - } - key := strings.TrimSpace(parts[0]) - config[key] = strings.TrimSpace(parts[1]) - } - - return config, nil -} - -// isValidWorkspace returns true iff the supplied path is the workspace root, defined by the presence of -// a file named WORKSPACE or WORKSPACE.bazel -// see https://github.com/bazelbuild/bazel/blob/8346ea4cfdd9fbd170d51a528fee26f912dad2d5/src/main/cpp/workspace_layout.cc#L37 -func isValidWorkspace(path string) bool { - info, err := os.Stat(path) - if err != nil { - return false - } - - return !info.IsDir() -} - -func findWorkspaceRoot(root string) string { - if isValidWorkspace(filepath.Join(root, "WORKSPACE")) { - return root - } - - if isValidWorkspace(filepath.Join(root, "WORKSPACE.bazel")) { - return root - } - - parentDirectory := filepath.Dir(root) - if parentDirectory == root { - return "" - } - - return findWorkspaceRoot(parentDirectory) -} - // TODO(go 1.18): remove backport of strings.Cut func cutString(s, sep string) (before, after string, found bool) { if i := strings.Index(s, sep); i >= 0 { @@ -315,7 +230,7 @@ func cutString(s, sep string) (before, after string, found bool) { return s, "", false } -func getBazelVersion() (string, error) { +func getBazelVersion(config config.Config) (string, error) { // Check in this order: // - env var "USE_BAZEL_VERSION" is set to a specific version. // - workspace_root/.bazeliskrc exists -> read contents, in contents: @@ -331,7 +246,7 @@ func getBazelVersion() (string, error) { // - workspace_root/.bazeliskrc exists -> read contents, in contents: // var "USE_BAZEL_FALLBACK_VERSION" is set to a fallback version format. // - fallback version format "silent:latest" - bazelVersion := GetEnvOrConfig("USE_BAZEL_VERSION") + bazelVersion := config.Get("USE_BAZEL_VERSION") if len(bazelVersion) != 0 { return bazelVersion, nil } @@ -341,7 +256,7 @@ func getBazelVersion() (string, error) { return "", fmt.Errorf("could not get working directory: %v", err) } - workspaceRoot := findWorkspaceRoot(workingDirectory) + workspaceRoot := ws.FindWorkspaceRoot(workingDirectory) if len(workspaceRoot) != 0 { bazelVersionPath := filepath.Join(workspaceRoot, ".bazelversion") if _, err := os.Stat(bazelVersionPath); err == nil { @@ -364,7 +279,7 @@ func getBazelVersion() (string, error) { } } - fallbackVersionFormat := GetEnvOrConfig("USE_BAZEL_FALLBACK_VERSION") + fallbackVersionFormat := config.Get("USE_BAZEL_FALLBACK_VERSION") fallbackVersionMode, fallbackVersion, hasFallbackVersionMode := cutString(fallbackVersionFormat, ":") if !hasFallbackVersionMode { fallbackVersionMode, fallbackVersion, hasFallbackVersionMode = "silent", fallbackVersionMode, true @@ -401,7 +316,7 @@ func parseBazelForkAndVersion(bazelForkAndVersion string) (string, string, error return bazelFork, bazelVersion, nil } -func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Repositories) (string, error) { +func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Repositories, config config.Config) (string, error) { bazelFork, bazelVersion, err := parseBazelForkAndVersion(bazelVersionString) if err != nil { return "", fmt.Errorf("could not parse Bazel fork and version: %v", err) @@ -412,12 +327,12 @@ func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Reposi return "", fmt.Errorf("could not resolve the version '%s' to an actual version number: %v", bazelVersion, err) } - bazelForkOrURL := dirForURL(GetEnvOrConfig(BaseURLEnv)) + bazelForkOrURL := dirForURL(config.Get(BaseURLEnv)) if len(bazelForkOrURL) == 0 { bazelForkOrURL = bazelFork } - bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, bazeliskHome, bazelForkOrURL, repos, downloader) + bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, bazeliskHome, bazelForkOrURL, repos, config, downloader) return bazelPath, err } @@ -430,11 +345,12 @@ func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Reposi // // downloads/metadata/[fork-or-url]/bazel-[version-os-etc] is a text file containing a hex sha256 of the contents of the downloaded bazel file. // downloads/sha256/[sha256]/bin/bazel[extension] contains the bazel with a particular sha256. -func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrURLDirName string, repos *Repositories, downloader DownloadFunc) (string, error) { +func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrURLDirName string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, error) { pathSegment, err := platforms.DetermineBazelFilename(version, false) if err != nil { return "", fmt.Errorf("could not determine path segment to use for Bazel binary: %v", err) } + destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() mappingPath := filepath.Join(bazeliskHome, "downloads", "metadata", bazelForkOrURLDirName, pathSegment) @@ -446,12 +362,12 @@ func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrUR } } - pathToBazelInCAS, downloadedDigest, err := downloadBazelToCAS(version, bazeliskHome, repos, downloader) + pathToBazelInCAS, downloadedDigest, err := downloadBazelToCAS(version, bazeliskHome, repos, config, downloader) if err != nil { return "", fmt.Errorf("failed to download bazel: %w", err) } - expectedSha256 := strings.ToLower(GetEnvOrConfig("BAZELISK_VERIFY_SHA256")) + expectedSha256 := strings.ToLower(config.Get("BAZELISK_VERIFY_SHA256")) if len(expectedSha256) > 0 { if expectedSha256 != downloadedDigest { return "", fmt.Errorf("%s has sha256=%s but need sha256=%s", pathToBazelInCAS, downloadedDigest, expectedSha256) @@ -479,7 +395,7 @@ func atomicWriteFile(path string, contents []byte, perm os.FileMode) error { return nil } -func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories, downloader DownloadFunc) (string, string, error) { +func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, string, error) { downloadsDir := filepath.Join(bazeliskHome, "downloads") temporaryDownloadDir := filepath.Join(downloadsDir, "_tmp") casDir := filepath.Join(bazeliskHome, "downloads", "sha256") @@ -492,12 +408,12 @@ func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories var tmpDestPath string var err error - baseURL := GetEnvOrConfig(BaseURLEnv) - formatURL := GetEnvOrConfig(FormatURLEnv) + baseURL := config.Get(BaseURLEnv) + formatURL := config.Get(FormatURLEnv) if baseURL != "" && formatURL != "" { return "", "", fmt.Errorf("cannot set %s and %s at once", BaseURLEnv, FormatURLEnv) } else if formatURL != "" { - tmpDestPath, err = repos.DownloadFromFormatURL(formatURL, version, temporaryDownloadDir, tmpDestFile) + tmpDestPath, err = repos.DownloadFromFormatURL(config, formatURL, version, temporaryDownloadDir, tmpDestFile) } else if baseURL != "" { tmpDestPath, err = repos.DownloadFromBaseURL(baseURL, version, temporaryDownloadDir, tmpDestFile) } else { @@ -575,24 +491,24 @@ func linkLocalBazel(baseDirectory string, bazelPath string) (string, error) { return destinationPath, nil } -func maybeDelegateToWrapperFromDir(bazel string, wd string, ignoreEnv bool) string { - if !ignoreEnv && GetEnvOrConfig(skipWrapperEnv) != "" { +func maybeDelegateToWrapperFromDir(bazel string, wd string, config config.Config) string { + if config.Get(skipWrapperEnv) != "" { return bazel } - root := findWorkspaceRoot(wd) + root := ws.FindWorkspaceRoot(wd) wrapper := filepath.Join(root, wrapperPath) if stat, err := os.Stat(wrapper); err == nil && !stat.Mode().IsDir() && stat.Mode().Perm()&0111 != 0 { return wrapper } if runtime.GOOS == "windows" { - powershellWrapper := filepath.Join(root, wrapperPath + ".ps1") + powershellWrapper := filepath.Join(root, wrapperPath+".ps1") if stat, err := os.Stat(powershellWrapper); err == nil && !stat.Mode().IsDir() { return powershellWrapper } - batchWrapper := filepath.Join(root, wrapperPath + ".bat") + batchWrapper := filepath.Join(root, wrapperPath+".bat") if stat, err := os.Stat(batchWrapper); err == nil && !stat.Mode().IsDir() { return batchWrapper } @@ -601,13 +517,13 @@ func maybeDelegateToWrapperFromDir(bazel string, wd string, ignoreEnv bool) stri return bazel } -func maybeDelegateToWrapper(bazel string) string { +func maybeDelegateToWrapper(bazel string, config config.Config) string { wd, err := os.Getwd() if err != nil { return bazel } - return maybeDelegateToWrapperFromDir(bazel, wd, false) + return maybeDelegateToWrapperFromDir(bazel, wd, config) } func prependDirToPathList(cmd *exec.Cmd, dir string) { @@ -629,8 +545,8 @@ func prependDirToPathList(cmd *exec.Cmd, dir string) { } } -func makeBazelCmd(bazel string, args []string, out io.Writer) *exec.Cmd { - execPath := maybeDelegateToWrapper(bazel) +func makeBazelCmd(bazel string, args []string, out io.Writer, config config.Config) *exec.Cmd { + execPath := maybeDelegateToWrapper(bazel, config) cmd := exec.Command(execPath, args...) cmd.Env = append(os.Environ(), skipWrapperEnv+"=true") @@ -648,8 +564,8 @@ func makeBazelCmd(bazel string, args []string, out io.Writer) *exec.Cmd { return cmd } -func runBazel(bazel string, args []string, out io.Writer) (int, error) { - cmd := makeBazelCmd(bazel, args, out) +func runBazel(bazel string, args []string, out io.Writer, config config.Config) (int, error) { + cmd := makeBazelCmd(bazel, args, out, config) err := cmd.Start() if err != nil { return 1, fmt.Errorf("could not start Bazel: %v", err) @@ -678,14 +594,14 @@ func runBazel(bazel string, args []string, out io.Writer) (int, error) { } // getIncompatibleFlags returns all incompatible flags for the current Bazel command in alphabetical order. -func getIncompatibleFlags(bazelPath, cmd string) ([]string, error) { - var incompatibleFlagsStr = GetEnvOrConfig("BAZELISK_INCOMPATIBLE_FLAGS") +func getIncompatibleFlags(bazelPath, cmd string, config config.Config) ([]string, error) { + var incompatibleFlagsStr = config.Get("BAZELISK_INCOMPATIBLE_FLAGS") if len(incompatibleFlagsStr) > 0 { return strings.Split(incompatibleFlagsStr, ","), nil } out := strings.Builder{} - if _, err := runBazel(bazelPath, []string{"help", cmd, "--short"}, &out); err != nil { + if _, err := runBazel(bazelPath, []string{"help", cmd, "--short"}, &out, config); err != nil { return nil, fmt.Errorf("unable to determine incompatible flags with binary %s: %v", bazelPath, err) } @@ -721,27 +637,27 @@ func insertArgs(baseArgs []string, newArgs []string) []string { func parseStartupOptions(baseArgs []string) []string { var result []string var bazelCommands = map[string]bool{ - "analyze-profile": true, - "aquery": true, - "build": true, + "analyze-profile": true, + "aquery": true, + "build": true, "canonicalize-flags": true, - "clean": true, - "coverage": true, - "cquery": true, - "dump": true, - "fetch": true, - "help": true, - "info": true, - "license": true, - "mobile-install": true, - "mod": true, - "print_action": true, - "query": true, - "run": true, - "shutdown": true, - "sync": true, - "test": true, - "version": true, + "clean": true, + "coverage": true, + "cquery": true, + "dump": true, + "fetch": true, + "help": true, + "info": true, + "license": true, + "mobile-install": true, + "mod": true, + "print_action": true, + "query": true, + "run": true, + "shutdown": true, + "sync": true, + "test": true, + "version": true, } // Arguments before a Bazel command are startup options. for _, arg := range baseArgs { @@ -753,15 +669,15 @@ func parseStartupOptions(baseArgs []string) []string { return result } -func shutdownIfNeeded(bazelPath string, startupOptions []string) { - bazeliskClean := GetEnvOrConfig("BAZELISK_SHUTDOWN") +func shutdownIfNeeded(bazelPath string, startupOptions []string, config config.Config) { + bazeliskClean := config.Get("BAZELISK_SHUTDOWN") if len(bazeliskClean) == 0 { return } args := append(startupOptions, "shutdown") fmt.Printf("bazel %s\n", strings.Join(args, " ")) - exitCode, err := runBazel(bazelPath, args, nil) + exitCode, err := runBazel(bazelPath, args, nil, config) fmt.Printf("\n") if err != nil { log.Fatalf("failed to run bazel shutdown: %v", err) @@ -772,15 +688,15 @@ func shutdownIfNeeded(bazelPath string, startupOptions []string) { } } -func cleanIfNeeded(bazelPath string, startupOptions []string) { - bazeliskClean := GetEnvOrConfig("BAZELISK_CLEAN") +func cleanIfNeeded(bazelPath string, startupOptions []string, config config.Config) { + bazeliskClean := config.Get("BAZELISK_CLEAN") if len(bazeliskClean) == 0 { return } args := append(startupOptions, "clean", "--expunge") fmt.Printf("bazel %s\n", strings.Join(args, " ")) - exitCode, err := runBazel(bazelPath, args, nil) + exitCode, err := runBazel(bazelPath, args, nil, config) fmt.Printf("\n") if err != nil { log.Fatalf("failed to run clean: %v", err) @@ -796,17 +712,17 @@ type ParentCommit struct { } type Commit struct { - SHA string `json:"sha"` + SHA string `json:"sha"` PARENTS []ParentCommit `json:"parents"` } type CompareResponse struct { - Commits []Commit `json:"commits"` - BaseCommit Commit `json:"base_commit"` - MergeBaseCommit Commit `json:"merge_base_commit"` + Commits []Commit `json:"commits"` + BaseCommit Commit `json:"base_commit"` + MergeBaseCommit Commit `json:"merge_base_commit"` } -func sendRequest(url string) (*http.Response, error) { +func sendRequest(url string, config config.Config) (*http.Response, error) { client := &http.Client{} req, err := http.NewRequest("GET", url, nil) @@ -814,7 +730,7 @@ func sendRequest(url string) (*http.Response, error) { return nil, err } - githubToken := GetEnvOrConfig("BAZELISK_GITHUB_TOKEN") + githubToken := config.Get("BAZELISK_GITHUB_TOKEN") if len(githubToken) != 0 { req.Header.Set("Authorization", fmt.Sprintf("token %s", githubToken)) } @@ -822,7 +738,7 @@ func sendRequest(url string) (*http.Response, error) { return client.Do(req) } -func getBazelCommitsBetween(goodCommit string, badCommit string) (string, []string, error) { +func getBazelCommitsBetween(goodCommit string, badCommit string, config config.Config) (string, []string, error) { commitList := make([]string, 0) page := 1 perPage := 250 // 250 is the maximum number of commits per page @@ -830,7 +746,7 @@ func getBazelCommitsBetween(goodCommit string, badCommit string) (string, []stri for { url := fmt.Sprintf("https://api.github.com/repos/bazelbuild/bazel/compare/%s...%s?page=%d&per_page=%d", goodCommit, badCommit, page, perPage) - response, err := sendRequest(url) + response, err := sendRequest(url, config) if err != nil { return goodCommit, nil, fmt.Errorf("Error fetching commit data: %v", err) } @@ -887,11 +803,11 @@ func getBazelCommitsBetween(goodCommit string, badCommit string) (string, []stri return goodCommit, commitList, nil } -func bisect(goodCommit string, badCommit string, args []string, bazeliskHome string, repos *Repositories) { +func bisect(goodCommit string, badCommit string, args []string, bazeliskHome string, repos *Repositories, config config.Config) { // 1. Get the list of commits between goodCommit and badCommit fmt.Printf("\n\n--- Getting the list of commits between %s and %s\n\n", goodCommit, badCommit) - goodCommit, commitList, err := getBazelCommitsBetween(goodCommit, badCommit) + goodCommit, commitList, err := getBazelCommitsBetween(goodCommit, badCommit, config) if err != nil { log.Fatalf("Failed to get commits: %v", err) os.Exit(1) @@ -899,7 +815,7 @@ func bisect(goodCommit string, badCommit string, args []string, bazeliskHome str // 2. Check if goodCommit is actually good fmt.Printf("\n\n--- Verifying if the given good Bazel commit (%s) is actually good\n\n", goodCommit) - bazelExitCode, err := testWithBazelAtCommit(goodCommit, args, bazeliskHome, repos) + bazelExitCode, err := testWithBazelAtCommit(goodCommit, args, bazeliskHome, repos, config) if err != nil { log.Fatalf("could not run Bazel: %v", err) os.Exit(1) @@ -916,8 +832,8 @@ func bisect(goodCommit string, badCommit string, args []string, bazeliskHome str for left < right { mid := (left + right) / 2 midCommit := commitList[mid] - fmt.Printf("\n\n--- Testing with Bazel built at %s, %d commits remaining...\n\n", midCommit, right -left) - bazelExitCode, err := testWithBazelAtCommit(midCommit, args, bazeliskHome, repos) + fmt.Printf("\n\n--- Testing with Bazel built at %s, %d commits remaining...\n\n", midCommit, right-left) + bazelExitCode, err := testWithBazelAtCommit(midCommit, args, bazeliskHome, repos, config) if err != nil { log.Fatalf("could not run Bazel: %v", err) os.Exit(1) @@ -943,16 +859,16 @@ func bisect(goodCommit string, badCommit string, args []string, bazeliskHome str os.Exit(0) } -func testWithBazelAtCommit(bazelCommit string, args []string, bazeliskHome string, repos *Repositories) (int, error) { - bazelPath, err := downloadBazel(bazelCommit, bazeliskHome, repos) +func testWithBazelAtCommit(bazelCommit string, args []string, bazeliskHome string, repos *Repositories, config config.Config) (int, error) { + bazelPath, err := downloadBazel(bazelCommit, bazeliskHome, repos, config) if err != nil { return 1, fmt.Errorf("could not download Bazel: %v", err) } startupOptions := parseStartupOptions(args) - shutdownIfNeeded(bazelPath, startupOptions) - cleanIfNeeded(bazelPath, startupOptions) + shutdownIfNeeded(bazelPath, startupOptions, config) + cleanIfNeeded(bazelPath, startupOptions, config) fmt.Printf("bazel %s\n", strings.Join(args, " ")) - bazelExitCode, err := runBazel(bazelPath, args, nil) + bazelExitCode, err := runBazel(bazelPath, args, nil, config) if err != nil { return -1, fmt.Errorf("could not run Bazel: %v", err) } @@ -960,16 +876,16 @@ func testWithBazelAtCommit(bazelCommit string, args []string, bazeliskHome strin } // migrate will run Bazel with each flag separately and report which ones are failing. -func migrate(bazelPath string, baseArgs []string, flags []string) { +func migrate(bazelPath string, baseArgs []string, flags []string, config config.Config) { var startupOptions = parseStartupOptions(baseArgs) // 1. Try with all the flags. args := insertArgs(baseArgs, flags) fmt.Printf("\n\n--- Running Bazel with all incompatible flags\n\n") - shutdownIfNeeded(bazelPath, startupOptions) - cleanIfNeeded(bazelPath, startupOptions) + shutdownIfNeeded(bazelPath, startupOptions, config) + cleanIfNeeded(bazelPath, startupOptions, config) fmt.Printf("bazel %s\n", strings.Join(args, " ")) - exitCode, err := runBazel(bazelPath, args, nil) + exitCode, err := runBazel(bazelPath, args, nil, config) if err != nil { log.Fatalf("could not run Bazel: %v", err) } @@ -981,10 +897,10 @@ func migrate(bazelPath string, baseArgs []string, flags []string) { // 2. Try with no flags, as a sanity check. args = baseArgs fmt.Printf("\n\n--- Running Bazel with no incompatible flags\n\n") - shutdownIfNeeded(bazelPath, startupOptions) - cleanIfNeeded(bazelPath, startupOptions) + shutdownIfNeeded(bazelPath, startupOptions, config) + cleanIfNeeded(bazelPath, startupOptions, config) fmt.Printf("bazel %s\n", strings.Join(args, " ")) - exitCode, err = runBazel(bazelPath, args, nil) + exitCode, err = runBazel(bazelPath, args, nil, config) if err != nil { log.Fatalf("could not run Bazel: %v", err) } @@ -999,10 +915,10 @@ func migrate(bazelPath string, baseArgs []string, flags []string) { for _, arg := range flags { args = insertArgs(baseArgs, []string{arg}) fmt.Printf("\n\n--- Running Bazel with %s\n\n", arg) - shutdownIfNeeded(bazelPath, startupOptions) - cleanIfNeeded(bazelPath, startupOptions) + shutdownIfNeeded(bazelPath, startupOptions, config) + cleanIfNeeded(bazelPath, startupOptions, config) fmt.Printf("bazel %s\n", strings.Join(args, " ")) - exitCode, err = runBazel(bazelPath, args, nil) + exitCode, err = runBazel(bazelPath, args, nil, config) if err != nil { log.Fatalf("could not run Bazel: %v", err) } diff --git a/core/core_test.go b/core/core_test.go index 9f1dea88..75424f28 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "runtime" "testing" + + "github.com/bazelbuild/bazelisk/config" ) func TestMaybeDelegateToNoWrapper(t *testing.T) { @@ -16,11 +18,11 @@ func TestMaybeDelegateToNoWrapper(t *testing.T) { } defer os.RemoveAll(tmpDir) - os.MkdirAll(tmpDir, os.ModeDir | 0700) + os.MkdirAll(tmpDir, os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "WORKSPACE"), []byte(""), 0600) ioutil.WriteFile(filepath.Join(tmpDir, "BUILD"), []byte(""), 0600) - entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, true) + entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, config.Null()) expected := "bazel_real" if entrypoint != expected { @@ -35,21 +37,21 @@ func TestMaybeDelegateToNoNonExecutableWrapper(t *testing.T) { if runtime.GOOS == "windows" { return } - + tmpDir, err := ioutil.TempDir("", "TestMaybeDelegateToNoNonExecutableWrapper") if err != nil { log.Fatal(err) } defer os.RemoveAll(tmpDir) - os.MkdirAll(tmpDir, os.ModeDir | 0700) + os.MkdirAll(tmpDir, os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "WORKSPACE"), []byte(""), 0600) ioutil.WriteFile(filepath.Join(tmpDir, "BUILD"), []byte(""), 0600) - os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir | 0700) + os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "tools", "bazel"), []byte(""), 0600) - entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, true) + entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, config.Null()) expected := "bazel_real" if entrypoint != expected { @@ -71,14 +73,14 @@ func TestMaybeDelegateToStandardWrapper(t *testing.T) { } defer os.RemoveAll(tmpDir) - os.MkdirAll(tmpDir, os.ModeDir | 0700) + os.MkdirAll(tmpDir, os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "WORKSPACE"), []byte(""), 0600) ioutil.WriteFile(filepath.Join(tmpDir, "BUILD"), []byte(""), 0600) - os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir | 0700) + os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "tools", "bazel"), []byte(""), 0700) - entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, true) + entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, config.Null()) expected := filepath.Join(tmpDir, "tools", "bazel") if entrypoint != expected { @@ -93,14 +95,14 @@ func TestMaybeDelegateToPowershellWrapper(t *testing.T) { } defer os.RemoveAll(tmpDir) - os.MkdirAll(tmpDir, os.ModeDir | 0700) + os.MkdirAll(tmpDir, os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "WORKSPACE"), []byte(""), 0600) ioutil.WriteFile(filepath.Join(tmpDir, "BUILD"), []byte(""), 0600) - os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir | 0700) + os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "tools", "bazel.ps1"), []byte(""), 0700) - entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, true) + entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, config.Null()) expected := filepath.Join(tmpDir, "tools", "bazel.ps1") // Only windows platforms use powershell wrappers @@ -120,14 +122,14 @@ func TestMaybeDelegateToBatchWrapper(t *testing.T) { } defer os.RemoveAll(tmpDir) - os.MkdirAll(tmpDir, os.ModeDir | 0700) + os.MkdirAll(tmpDir, os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "WORKSPACE"), []byte(""), 0600) ioutil.WriteFile(filepath.Join(tmpDir, "BUILD"), []byte(""), 0600) - os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir | 0700) + os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "tools", "bazel.bat"), []byte(""), 0700) - entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, true) + entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, config.Null()) expected := filepath.Join(tmpDir, "tools", "bazel.bat") // Only windows platforms use batch wrappers @@ -147,15 +149,15 @@ func TestMaybeDelegateToPowershellOverBatchWrapper(t *testing.T) { } defer os.RemoveAll(tmpDir) - os.MkdirAll(tmpDir, os.ModeDir | 0700) + os.MkdirAll(tmpDir, os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "WORKSPACE"), []byte(""), 0600) ioutil.WriteFile(filepath.Join(tmpDir, "BUILD"), []byte(""), 0600) - os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir | 0700) + os.MkdirAll(filepath.Join(tmpDir, "tools"), os.ModeDir|0700) ioutil.WriteFile(filepath.Join(tmpDir, "tools", "bazel.ps1"), []byte(""), 0700) ioutil.WriteFile(filepath.Join(tmpDir, "tools", "bazel.bat"), []byte(""), 0700) - entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, true) + entrypoint := maybeDelegateToWrapperFromDir("bazel_real", tmpDir, config.Null()) expected := filepath.Join(tmpDir, "tools", "bazel.ps1") // Only windows platforms use powershell or batch wrappers diff --git a/core/repositories.go b/core/repositories.go index 7b5b4be6..e11e4460 100644 --- a/core/repositories.go +++ b/core/repositories.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/bazelbuild/bazelisk/config" "github.com/bazelbuild/bazelisk/httputil" "github.com/bazelbuild/bazelisk/platforms" "github.com/bazelbuild/bazelisk/versions" @@ -235,7 +236,7 @@ func (r *Repositories) DownloadFromBaseURL(baseURL, version, destDir, destFile s return httputil.DownloadBinary(url, destDir, destFile) } -func BuildURLFromFormat(formatURL, version string) (string, error) { +func BuildURLFromFormat(config config.Config, formatURL, version string) (string, error) { osName, err := platforms.DetermineOperatingSystem() if err != nil { return "", err @@ -261,7 +262,7 @@ func BuildURLFromFormat(formatURL, version string) (string, error) { case 'e': b.WriteString(platforms.DetermineExecutableFilenameSuffix()) case 'h': - b.WriteString(GetEnvOrConfig("BAZELISK_VERIFY_SHA256")) + b.WriteString(config.Get("BAZELISK_VERIFY_SHA256")) case 'm': b.WriteString(machineName) case 'o': @@ -281,12 +282,12 @@ func BuildURLFromFormat(formatURL, version string) (string, error) { } // DownloadFromFormatURL can download Bazel binaries from a specific URL while ignoring the predefined repositories. -func (r *Repositories) DownloadFromFormatURL(formatURL, version, destDir, destFile string) (string, error) { +func (r *Repositories) DownloadFromFormatURL(config config.Config, formatURL, version, destDir, destFile string) (string, error) { if formatURL == "" { return "", fmt.Errorf("%s is not set", FormatURLEnv) } - url, err := BuildURLFromFormat(formatURL, version) + url, err := BuildURLFromFormat(config, formatURL, version) if err != nil { return "", err } diff --git a/core/repositories_test.go b/core/repositories_test.go index f6442446..27ee65bc 100644 --- a/core/repositories_test.go +++ b/core/repositories_test.go @@ -3,9 +3,9 @@ package core import ( "errors" "fmt" - "os" "testing" + "github.com/bazelbuild/bazelisk/config" "github.com/bazelbuild/bazelisk/platforms" ) @@ -24,18 +24,10 @@ func TestBuildURLFromFormat(t *testing.T) { suffix := platforms.DetermineExecutableFilenameSuffix() - previousSha256, hadSha256 := os.LookupEnv("BAZELISK_VERIFY_SHA256") sha256 := "SomeSha256ValueThatIsIrrelevant" - if err := os.Setenv("BAZELISK_VERIFY_SHA256", sha256); err != nil { - t.Fatalf("Failed to set BAZELISK_VERIFY_SHA256") - } - defer func() { - if hadSha256 { - os.Setenv("BAZELISK_VERIFY_SHA256", previousSha256) - } else { - os.Unsetenv("BAZELISK_VERIFY_SHA256") - } - }() + config := config.Static(map[string]string{ + "BAZELISK_VERIFY_SHA256": sha256, + }) type test struct { format string @@ -65,7 +57,7 @@ func TestBuildURLFromFormat(t *testing.T) { } for _, tc := range tests { - got, err := BuildURLFromFormat(tc.format, version) + got, err := BuildURLFromFormat(config, tc.format, version) if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tc.wantErr) { if got != "" { t.Errorf("format '%s': got non-empty '%s' on error", tc.format, got) diff --git a/ws/BUILD b/ws/BUILD new file mode 100644 index 00000000..8b3ebbf7 --- /dev/null +++ b/ws/BUILD @@ -0,0 +1,11 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["ws.go"], + importpath = "github.com/bazelbuild/bazelisk/ws", + visibility = ["//visibility:public"], + deps = [ + "@com_github_mitchellh_go_homedir//:go_default_library", + ], +) diff --git a/ws/ws.go b/ws/ws.go new file mode 100644 index 00000000..653090de --- /dev/null +++ b/ws/ws.go @@ -0,0 +1,36 @@ +package ws + +import ( + "os" + "path/filepath" +) + +// FindWorkspaceRoot returns the root directory of the Bazel workspace in which the passed root exists, if any. +func FindWorkspaceRoot(root string) string { + if isValidWorkspace(filepath.Join(root, "WORKSPACE")) { + return root + } + + if isValidWorkspace(filepath.Join(root, "WORKSPACE.bazel")) { + return root + } + + parentDirectory := filepath.Dir(root) + if parentDirectory == root { + return "" + } + + return FindWorkspaceRoot(parentDirectory) +} + +// isValidWorkspace returns true iff the supplied path is the workspace root, defined by the presence of +// a file named WORKSPACE or WORKSPACE.bazel +// see https://github.com/bazelbuild/bazel/blob/8346ea4cfdd9fbd170d51a528fee26f912dad2d5/src/main/cpp/workspace_layout.cc#L37 +func isValidWorkspace(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + + return !info.IsDir() +} From ac2ae9dad9e79f51485aefa836ca85bb01d9c4a5 Mon Sep 17 00:00:00 2001 From: Zerun Zhao Date: Thu, 2 Mar 2023 19:24:59 -0800 Subject: [PATCH 17/22] add fallback to default release bazel url --- bazelisk.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bazelisk.py b/bazelisk.py index 8a967c27..889379fd 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -316,6 +316,12 @@ def download_bazel_into_directory(version, is_commit, directory): sys.stderr.write( "The Bazel mirror does not have a checksum file; skipping checksum verification." ) + if "https://releases.bazel.build" not in bazel_url: + (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups() + fallback_url="https://releases.bazel.build/{}/{}/{}".format( + version, rc if rc else "release", bazel_filename + ) + download(fallback_url, destination_path) return destination_path raise e with open(sha256_path, "r") as sha_file: @@ -326,16 +332,17 @@ def download_bazel_into_directory(version, is_commit, directory): sha256_hash.update(byte_block) actual_hash = sha256_hash.hexdigest() if actual_hash != expected_hash: - os.remove(destination_path) os.remove(sha256_path) print( - "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Please try again.".format( + "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Fallback to default releases.bazel build url.".format( expected_hash, actual_hash ) ) - # Exiting with a special exit code not used by Bazel, so the calling process may retry based on that. - # https://docs.bazel.build/versions/0.21.0/guide.html#what-exit-code-will-i-get - sys.exit(22) + (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups() + fallback_url="https://releases.bazel.build/{}/{}/{}".format( + version, rc if rc else "release", bazel_filename + ) + download(fallback_url, destination_path) return destination_path From f49ad68e0539b50adb6d2aa985d49ee9133dc469 Mon Sep 17 00:00:00 2001 From: Zerun Zhao Date: Thu, 2 Mar 2023 19:32:26 -0800 Subject: [PATCH 18/22] add fallback to default release bazel url --- bazelisk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazelisk.py b/bazelisk.py index 889379fd..ed997ce7 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -334,7 +334,7 @@ def download_bazel_into_directory(version, is_commit, directory): if actual_hash != expected_hash: os.remove(sha256_path) print( - "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Fallback to default releases.bazel build url.".format( + "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Fallback to default releases.bazel.build url.".format( expected_hash, actual_hash ) ) From bb3d6f75b2cdee321de4aff30a7ebe8adc686ea2 Mon Sep 17 00:00:00 2001 From: Zerun Zhao Date: Fri, 3 Mar 2023 10:40:15 -0800 Subject: [PATCH 19/22] update --- bazelisk.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bazelisk.py b/bazelisk.py index ed997ce7..5f0cf8c3 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -317,11 +317,13 @@ def download_bazel_into_directory(version, is_commit, directory): "The Bazel mirror does not have a checksum file; skipping checksum verification." ) if "https://releases.bazel.build" not in bazel_url: - (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups() - fallback_url="https://releases.bazel.build/{}/{}/{}".format( - version, rc if rc else "release", bazel_filename - ) - download(fallback_url, destination_path) + matched=re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version) + if matched: + (version, rc) = matched.groups() + fallback_url="https://releases.bazel.build/{}/{}/{}".format( + version, rc if rc else "release", bazel_filename + ) + download(fallback_url, destination_path) return destination_path raise e with open(sha256_path, "r") as sha_file: From 826f2e1b639a37594c4caf1ff788526d9c9b2f2d Mon Sep 17 00:00:00 2001 From: Zerun Zhao Date: Fri, 3 Mar 2023 13:27:40 -0800 Subject: [PATCH 20/22] update --- bazelisk.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/bazelisk.py b/bazelisk.py index 5f0cf8c3..6fff50c0 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -323,8 +323,14 @@ def download_bazel_into_directory(version, is_commit, directory): fallback_url="https://releases.bazel.build/{}/{}/{}".format( version, rc if rc else "release", bazel_filename ) - download(fallback_url, destination_path) - return destination_path + download(fallback_url+".sha256", sha256_path) + os.remove(destination_path) + download(fallback_url,destination_path) + os.chmod(destination_path, 0o755) + else: + return destination_path + else: + return destination_path raise e with open(sha256_path, "r") as sha_file: expected_hash = sha_file.read().split()[0] @@ -340,11 +346,16 @@ def download_bazel_into_directory(version, is_commit, directory): expected_hash, actual_hash ) ) - (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups() - fallback_url="https://releases.bazel.build/{}/{}/{}".format( - version, rc if rc else "release", bazel_filename - ) - download(fallback_url, destination_path) + matched=re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version) + if matched: + (version, rc) = matched.groups() + fallback_url="https://releases.bazel.build/{}/{}/{}".format( + version, rc if rc else "release", bazel_filename + ) + download(fallback_url+".sha256", sha256_path) + os.remove(destination_path) + download(fallback_url,destination_path) + os.chmod(destination_path, 0o755) return destination_path From 8b9e8d63c53b197f56a5e230e90ac4e8423b10b0 Mon Sep 17 00:00:00 2001 From: Zerun Zhao Date: Thu, 16 Mar 2023 11:07:53 -0700 Subject: [PATCH 21/22] address comments --- bazelisk.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/bazelisk.py b/bazelisk.py index 6fff50c0..a17ec652 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -308,6 +308,8 @@ def download_bazel_into_directory(version, is_commit, directory): sha256_path = destination_path + ".sha256" expected_hash = "" + matcher=re.compile(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?") + matched=matcher.match(version) if not os.path.exists(sha256_path): try: download(bazel_url + ".sha256", sha256_path) @@ -316,21 +318,20 @@ def download_bazel_into_directory(version, is_commit, directory): sys.stderr.write( "The Bazel mirror does not have a checksum file; skipping checksum verification." ) - if "https://releases.bazel.build" not in bazel_url: - matched=re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version) - if matched: - (version, rc) = matched.groups() - fallback_url="https://releases.bazel.build/{}/{}/{}".format( - version, rc if rc else "release", bazel_filename - ) - download(fallback_url+".sha256", sha256_path) + if "https://releases.bazel.build" in bazel_url or not matched: + return destination_path + if matched: + (version, rc) = matched.groups() + fallback_url="https://releases.bazel.build/{}/{}/{}".format( + version, rc if rc else "release", bazel_filename + ) + try: + download("{}.sha256".format(fallback_url), sha256_path) os.remove(destination_path) download(fallback_url,destination_path) - os.chmod(destination_path, 0o755) - else: + except HTTPError: return destination_path - else: - return destination_path + os.chmod(destination_path, 0o755) raise e with open(sha256_path, "r") as sha_file: expected_hash = sha_file.read().split()[0] @@ -341,20 +342,22 @@ def download_bazel_into_directory(version, is_commit, directory): actual_hash = sha256_hash.hexdigest() if actual_hash != expected_hash: os.remove(sha256_path) + os.remove(destination_path) print( "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Fallback to default releases.bazel.build url.".format( expected_hash, actual_hash ) ) - matched=re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version) if matched: (version, rc) = matched.groups() fallback_url="https://releases.bazel.build/{}/{}/{}".format( version, rc if rc else "release", bazel_filename ) - download(fallback_url+".sha256", sha256_path) - os.remove(destination_path) - download(fallback_url,destination_path) + try: + download("{}.sha256".format(fallback_url), sha256_path) + download(fallback_url,destination_path) + except HTTPError: + exit(22) os.chmod(destination_path, 0o755) return destination_path From 2e46381c7e18ba45b80b65c056bc63283329146d Mon Sep 17 00:00:00 2001 From: Zerun Zhao Date: Wed, 5 Jul 2023 10:27:57 -0700 Subject: [PATCH 22/22] run black --- bazelisk.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bazelisk.py b/bazelisk.py index a17ec652..be2cd1cd 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -180,8 +180,7 @@ def get_version_history(bazelisk_directory): ), # This only handles versions with numeric components, but that is fine # since prerelease versions have been excluded. - key=lambda version: tuple(int(component) - for component in version.split('.')), + key=lambda version: tuple(int(component) for component in version.split(".")), reverse=True, ) @@ -308,8 +307,8 @@ def download_bazel_into_directory(version, is_commit, directory): sha256_path = destination_path + ".sha256" expected_hash = "" - matcher=re.compile(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?") - matched=matcher.match(version) + matcher = re.compile(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?") + matched = matcher.match(version) if not os.path.exists(sha256_path): try: download(bazel_url + ".sha256", sha256_path) @@ -322,13 +321,13 @@ def download_bazel_into_directory(version, is_commit, directory): return destination_path if matched: (version, rc) = matched.groups() - fallback_url="https://releases.bazel.build/{}/{}/{}".format( + fallback_url = "https://releases.bazel.build/{}/{}/{}".format( version, rc if rc else "release", bazel_filename ) try: download("{}.sha256".format(fallback_url), sha256_path) os.remove(destination_path) - download(fallback_url,destination_path) + download(fallback_url, destination_path) except HTTPError: return destination_path os.chmod(destination_path, 0o755) @@ -350,12 +349,12 @@ def download_bazel_into_directory(version, is_commit, directory): ) if matched: (version, rc) = matched.groups() - fallback_url="https://releases.bazel.build/{}/{}/{}".format( + fallback_url = "https://releases.bazel.build/{}/{}/{}".format( version, rc if rc else "release", bazel_filename ) try: download("{}.sha256".format(fallback_url), sha256_path) - download(fallback_url,destination_path) + download(fallback_url, destination_path) except HTTPError: exit(22) os.chmod(destination_path, 0o755)