diff --git a/README.md b/README.md index 5aac7fb..1467698 100644 --- a/README.md +++ b/README.md @@ -179,8 +179,24 @@ rust_repo( git_revision = "1.0.86", ) ``` -(`sync --import` translates `git+https://github.com/...` lockfile sources -automatically.) +`git_repo` also takes a full URL, for any forge: +```python +rust_repo( + name = "thing", + crate = "thing", + version = "0.3.0", + git_repo = "https://gitlab.com/group/thing", + git_revision = "v0.3.0", +) +``` +Most forges serve `/archive/.tar.gz`, which is what github, gitea and +its forks, and sourcehut all do. GitLab serves +`/-/archive//-.tar.gz`, and is recognised by its host. +GitLab running somewhere its name does not say needs `git_forge = "gitlab"`. +A forge serving neither scheme needs `download = ...`, naming any rule that +produces the crate's source. + +`sync --import` translates `git+https://` lockfile sources from any host. `rust_library` builds an `rlib` by default; `crate_type` also supports `proc-macro`, `dylib`, `cdylib` and `staticlib` for compiler plugins and diff --git a/build_defs/rust.build_defs b/build_defs/rust.build_defs index fe4fc4b..066e759 100644 --- a/build_defs/rust.build_defs +++ b/build_defs/rust.build_defs @@ -1243,8 +1243,64 @@ def rust_crate_download(name:str, crate:str, version:str, hashes:list=None, labe ) +def forge_archive_url(repo:str, revision:str, host:str="github.com", forge:str=""): + """The URL of a source archive for one revision of a repository. + + Most forges serve `/archive/.tar.gz`: github, gitea and its forks + (codeberg), and sourcehut all do. GitLab is the exception and serves + `/-/archive//-.tar.gz`; its `/archive/` path answers + 403, so a gitlab repo declared as though it were github fails at download + with no hint as to why. + + The host decides the scheme, which cannot work for a self-hosted forge + whose name says nothing about what it runs. `forge` says it outright. + + Args: + repo: Either `owner/repo`, which means github.com, or a full URL to + the repository on any forge. + revision: Commit sha or tag to fetch. + host: Forge host, when repo is the shorthand. + forge: 'github' or 'gitlab' to say which archive scheme the forge + serves. Inferred from the host when empty, which is right for + gitlab.com and wrong for gitlab running somewhere else. + """ + if repo.startswith("https://") or repo.startswith("http://"): + base = repo.rstrip("/") + if base.endswith(".git"): + base = base[:-4] + # Split on the whole path rather than with a maxsplit, which this + # dialect does not take. + rest = base.replace("https://", "").replace("http://", "") + segments = rest.split("/") + forge_host = segments[0] + path = "/".join(segments[1:]) + else: + forge_host = host + path = repo.strip("/") + base = f"https://{forge_host}/{path}" + kind = forge + if not kind: + kind = "gitlab" if forge_host == "gitlab.com" or forge_host.startswith("gitlab.") else "github" + if kind == "gitlab": + # The tarball is named for the project, which is the last path + # segment, not for the whole path: a repo under a group nests. The + # name itself is cosmetic, gitlab serves the archive whatever it says. + project = path.split("/")[-1] + # The revision is its own path segment here, followed by that name, so + # a tag with a slash in it has to be escaped or gitlab reads the part + # before the slash as the whole ref. sequoia tags releases + # openpgp/v2.4.1 because one repository holds several crates. It + # happens to resolve unescaped today only because no branch is named + # openpgp; one appearing would silently fetch a different tree. + ref = revision.replace("/", "%2F") + return f"{base}/-/archive/{ref}/{project}-{revision}.tar.gz" + # Github takes the ref as the last segment before the extension, where a + # slash needs no escaping and escaping it would break the URL. + return f"{base}/archive/{revision}.tar.gz" + + def rust_git_download(name:str, crate:str, version:str, repo:str, revision:str, hashes:list=None, - host:str="github.com", subdir:str="", visibility:list=None): + host:str="github.com", forge:str="", subdir:str="", visibility:list=None): """Downloads a crate's source from a git forge archive (for forks and unpublished revisions), normalized to the {crate}-{version} layout that rust_repo expects. The go-rules `download` pattern: the crate keeps its @@ -1254,15 +1310,17 @@ def rust_git_download(name:str, crate:str, version:str, repo:str, revision:str, name: Name of the rule. crate: Crate name (canonical, as dependents refer to it). version: Version dependents resolve against. - repo: owner/repo on the forge. + repo: `owner/repo` on github, or a full repository URL on any forge. revision: Commit sha or tag to fetch. hashes: Optional archive hashes to verify. - host: Forge host serving /archive/ tarballs (github.com, gitlab works too). + host: Forge host, when repo is the shorthand. + forge: Archive scheme to use, 'github' or 'gitlab'. Inferred from the + host when empty. subdir: Path of the crate within the repo, for workspace members. """ dl = remote_file( name = tag(name, "git"), - url = f"https://{host}/{repo}/archive/{revision}.tar.gz", + url = forge_archive_url(repo, revision, host, forge), hashes = hashes, extract = True, ) @@ -1333,6 +1391,7 @@ def rust_repo( download: str = None, git_repo: str = "", git_revision: str = "", + git_forge: str = "", platforms: list = None, ): """Downloads a crate and generates BUILD files as a subrepo. @@ -1378,9 +1437,13 @@ def rust_repo( download: A rule providing the crate source (a {crate}-{version} dir with Cargo.toml at its root) instead of the crates.io tarball — e.g. a rust_git_download of a fork. - git_repo: owner/repo shorthand: fetch the source from this git forge - repository instead of crates.io. + git_repo: `owner/repo` on github, or a full repository URL on any + forge, to fetch the source from rather than crates.io. git_revision: Commit sha or tag for git_repo. + git_forge: Archive scheme the forge serves, 'github' or 'gitlab'. + Inferred from the host when empty, which is right for + gitlab.com and wrong for gitlab self-hosted somewhere its + name does not say. platforms: The operating systems (in Rust's `target_os` vocabulary) whose resolution reaches this crate. Omitted for the overwhelming majority of crates, which build anywhere; @@ -1436,6 +1499,7 @@ def rust_repo( version = version, repo = git_repo, revision = git_revision or version, + forge = git_forge, hashes = hashes, ) elif not download: diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index fab5990..e69c033 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -47,7 +47,6 @@ rather than by release: [#15](https://github.com/becomeliminal/rust-rules/issues/15) remote execution audit, [#16](https://github.com/becomeliminal/rust-rules/issues/16) musl and embedded - **Missing capability:** - [#21](https://github.com/becomeliminal/rust-rules/issues/21) git forges, [#23](https://github.com/becomeliminal/rust-rules/issues/23) cross-compiling C, [#24](https://github.com/becomeliminal/rust-rules/issues/24) channels, [#25](https://github.com/becomeliminal/rust-rules/issues/25) cbindgen, @@ -82,7 +81,7 @@ rather than by release: | **Add a dependency in one command**
No repin step | **yes**. lock --add crate@req, re-solves and declares | **partial**. cargo add then a repin of crate_universe | **yes**. cargo add | | **Upgrade to the newest compatible versions**
The cargo update equivalent | **yes**. lock --upgrade for everything, or named crates; stays inside each declaration's compatibility range | **no**. Re-run the generator against new requirements | **yes**. cargo update | | **Import an existing Cargo.lock**
Adopting a repo that already uses cargo | **yes**. sync --import, and --import-workspace for BUILD files too | **yes**. crate_universe consumes Cargo.toml directly | **yes**. Native | -| **Git and fork dependencies**
Pinned revision instead of crates.io | **partial**. github archive URLs; other forges need download= | **yes**. Supported | **yes**. Native | +| **Git and fork dependencies**
Pinned revision instead of crates.io | **yes**. Any forge serving source archives, github and gitlab schemes both | **yes**. Supported | **yes**. Native | | **Private or alternative registries**
Registry auth | **no**. On demand only; forks and download= cover the cases | **yes**. Via cargo | **yes**. Native | | **Vendored source overrides**
Patch or replace a crate's source | **yes**. download= overrides the source, patch= applies patches to it | **yes**. annotations and patches | **yes**. [patch] and [replace] | diff --git a/test/forge/BUILD b/test/forge/BUILD new file mode 100644 index 0000000..9899eee --- /dev/null +++ b/test/forge/BUILD @@ -0,0 +1,63 @@ +subinclude("//build_defs:rust") + +# Where a crate's source comes from when it is not crates.io. +# +# Most forges serve /archive/.tar.gz: github, gitea and its forks such +# as codeberg, and sourcehut all do. GitLab does not. Its /archive/ path +# answers 403 and the tarball lives at /-/archive//-.tar.gz, +# so a gitlab repo declared as though it were github fails at download with +# nothing to say why. +# +# These run when the package is parsed, so a change to the URL scheme fails +# the build rather than a fetch. +_forge_cases = [ + # repo, forge, expected URL + ["owner/repo", "", "https://github.com/owner/repo/archive/abc123.tar.gz"], + [ + "https://github.com/owner/repo", + "", + "https://github.com/owner/repo/archive/abc123.tar.gz", + ], + # A trailing .git is how a Cargo.lock spells the same repository. + [ + "https://codeberg.org/owner/repo.git", + "", + "https://codeberg.org/owner/repo/archive/abc123.tar.gz", + ], + # gitlab.com is recognised by its host, and the tarball is named for the + # project rather than the whole path, so a repo inside a group nests. + [ + "https://gitlab.com/group/sub/proj", + "", + "https://gitlab.com/group/sub/proj/-/archive/abc123/proj-abc123.tar.gz", + ], + # A forge whose name says nothing about what it runs has to be told. + [ + "https://git.corp.example/team/proj", + "gitlab", + "https://git.corp.example/team/proj/-/archive/abc123/proj-abc123.tar.gz", + ], + [ + "https://git.corp.example/team/proj", + "", + "https://git.corp.example/team/proj/archive/abc123.tar.gz", + ], +] + +# A tag can contain a slash. On gitlab the revision is its own path segment, +# so it has to be escaped: sequoia tags releases openpgp/v2.4.1 because one +# repository holds several crates. +_slash = forge_archive_url("https://gitlab.com/sequoia-pgp/sequoia", "openpgp/v2.4.1", "github.com", "") +_want_slash = "https://gitlab.com/sequoia-pgp/sequoia/-/archive/openpgp%2Fv2.4.1/sequoia-openpgp/v2.4.1.tar.gz" +if _slash != _want_slash: + fail("slashed gitlab tag: got %s, want %s" % (_slash, _want_slash)) + +# Github takes the ref as the last segment, where a slash needs no escaping. +_ghslash = forge_archive_url("owner/repo", "release/1.0", "github.com", "") +if _ghslash != "https://github.com/owner/repo/archive/release/1.0.tar.gz": + fail("slashed github ref should not be escaped: %s" % _ghslash) + +for _case in _forge_cases: + _got = forge_archive_url(_case[0], "abc123", "github.com", _case[1]) + if _got != _case[2]: + fail("forge_archive_url(%s, forge=%s) = %s, want %s" % (_case[0], _case[1], _got, _case[2])) diff --git a/tools/please_rust/src/sync.rs b/tools/please_rust/src/sync.rs index 2518e82..5ce7835 100644 --- a/tools/please_rust/src/sync.rs +++ b/tools/please_rust/src/sync.rs @@ -780,15 +780,17 @@ fn import_cargo_lock(path: &Path, decls: &mut Vec) -> Result<()> { let (url, frag) = rest.split_once('#').unwrap_or((rest, "")); let url = url.split('?').next().unwrap_or(url); if let Some(path) = url.strip_prefix("https://github.com/") { + // The shorthand, because that is what the rule has always + // recorded for github and what its declarations look like. git_repo = path.trim_end_matches(".git").to_string(); - git_revision = frag.to_string(); } else { - eprintln!( - "warning: {} uses a non-github git source ({}); declare it manually with rust_repo(download = ...)", - name, url - ); - continue; + // Any other forge is recorded as the URL it was cloned from. + // The rule derives the archive scheme from the host, which is + // right except for gitlab hosted somewhere its name does not + // say; those need git_forge = "gitlab" adding by hand. + git_repo = url.trim_end_matches(".git").to_string(); } + git_revision = frag.to_string(); if git_revision.is_empty() { eprintln!( "warning: {} git source has no pinned revision, skipping", @@ -2303,8 +2305,13 @@ fresh = { version = "2", features = ["extra"] } let forked = decls.iter().find(|d| d.crate_name == "forked").unwrap(); assert_eq!(forked.git_repo, "owner/forked"); assert_eq!(forked.git_revision, "abcdef123456"); - // Non-github git and local path crates skipped - assert!(!decls.iter().any(|d| d.crate_name == "elsewhere")); + // A git source on any other forge is imported as the URL it was + // cloned from, rather than skipped with a note to write the rule by + // hand. The rule derives the archive scheme from the host. + let elsewhere = decls.iter().find(|d| d.crate_name == "elsewhere").unwrap(); + assert_eq!(elsewhere.git_repo, "https://gitlab.example.com/x/y"); + assert_eq!(elsewhere.git_revision, "deadbeef"); + // A path dependency has no source to fetch, so it stays skipped. assert!(!decls.iter().any(|d| d.crate_name == "local_thing")); }