From a834b0e1a01107a9cdd6e459b2133d6f93ee34a4 Mon Sep 17 00:00:00 2001 From: Jack Marsh Date: Mon, 24 Aug 2026 11:52:17 +0100 Subject: [PATCH 1/2] Fetch a crate from any forge, not only github rust_repo already covered both sources in one rule: a crate comes from crates.io, or from a git forge with git_repo and git_revision. The forge half was github only, and the docstring's claim that gitlab worked too was never true. GitLab's /archive/ path answers 403; the tarball is at /-/archive//-.tar.gz. git_repo now takes a full URL as well as the owner/repo shorthand, and the archive URL is built per forge scheme. Verified against each forge rather than assumed: github.com /archive/.tar.gz 200 codeberg /archive/.tar.gz 200 sr.ht /archive/.tar.gz 200 gitlab.com /-/archive//-.tar.gz 200 gitlab.com /archive/.tar.gz 403 The scheme is inferred from the host, which cannot work for gitlab running somewhere its name does not say, so git_forge names it outright. sync --import records any other forge as the URL it was cloned from rather than warning and telling the reader to write the rule by hand. test/forge asserts the URLs while the package is parsed, so a change to the scheme fails the build rather than a fetch. It caught the first version of this: split(sep, 1) does not take a maxsplit in this dialect, so every URL input was broken. Nothing here needs git. A forge serving neither scheme still needs download=, which is the escape hatch go-rules has for the same case. --- README.md | 20 +++++++++-- build_defs/rust.build_defs | 66 +++++++++++++++++++++++++++++++---- docs/COMPARISON.md | 3 +- test/forge/BUILD | 50 ++++++++++++++++++++++++++ tools/please_rust/src/sync.rs | 23 +++++++----- 5 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 test/forge/BUILD diff --git a/README.md b/README.md index 5aac7fb..1467698 100644 --- a/README.md +++ b/README.md @@ -179,8 +179,24 @@ rust_repo( git_revision = "1.0.86", ) ``` -(`sync --import` translates `git+https://github.com/...` lockfile sources -automatically.) +`git_repo` also takes a full URL, for any forge: +```python +rust_repo( + name = "thing", + crate = "thing", + version = "0.3.0", + git_repo = "https://gitlab.com/group/thing", + git_revision = "v0.3.0", +) +``` +Most forges serve `/archive/.tar.gz`, which is what github, gitea and +its forks, and sourcehut all do. GitLab serves +`/-/archive//-.tar.gz`, and is recognised by its host. +GitLab running somewhere its name does not say needs `git_forge = "gitlab"`. +A forge serving neither scheme needs `download = ...`, naming any rule that +produces the crate's source. + +`sync --import` translates `git+https://` lockfile sources from any host. `rust_library` builds an `rlib` by default; `crate_type` also supports `proc-macro`, `dylib`, `cdylib` and `staticlib` for compiler plugins and diff --git a/build_defs/rust.build_defs b/build_defs/rust.build_defs index fe4fc4b..b265b6b 100644 --- a/build_defs/rust.build_defs +++ b/build_defs/rust.build_defs @@ -1243,8 +1243,54 @@ def rust_crate_download(name:str, crate:str, version:str, hashes:list=None, labe ) +def forge_archive_url(repo:str, revision:str, host:str="github.com", forge:str=""): + """The URL of a source archive for one revision of a repository. + + Most forges serve `/archive/.tar.gz`: github, gitea and its forks + (codeberg), and sourcehut all do. GitLab is the exception and serves + `/-/archive//-.tar.gz`; its `/archive/` path answers + 403, so a gitlab repo declared as though it were github fails at download + with no hint as to why. + + The host decides the scheme, which cannot work for a self-hosted forge + whose name says nothing about what it runs. `forge` says it outright. + + Args: + repo: Either `owner/repo`, which means github.com, or a full URL to + the repository on any forge. + revision: Commit sha or tag to fetch. + host: Forge host, when repo is the shorthand. + forge: 'github' or 'gitlab' to say which archive scheme the forge + serves. Inferred from the host when empty, which is right for + gitlab.com and wrong for gitlab running somewhere else. + """ + if repo.startswith("https://") or repo.startswith("http://"): + base = repo.rstrip("/") + if base.endswith(".git"): + base = base[:-4] + # Split on the whole path rather than with a maxsplit, which this + # dialect does not take. + rest = base.replace("https://", "").replace("http://", "") + segments = rest.split("/") + forge_host = segments[0] + path = "/".join(segments[1:]) + else: + forge_host = host + path = repo.strip("/") + base = f"https://{forge_host}/{path}" + kind = forge + if not kind: + kind = "gitlab" if forge_host == "gitlab.com" or forge_host.startswith("gitlab.") else "github" + if kind == "gitlab": + # The tarball is named for the project, which is the last path + # segment, not for the whole path: a repo under a group nests. + project = path.split("/")[-1] + return f"{base}/-/archive/{revision}/{project}-{revision}.tar.gz" + return f"{base}/archive/{revision}.tar.gz" + + def rust_git_download(name:str, crate:str, version:str, repo:str, revision:str, hashes:list=None, - host:str="github.com", subdir:str="", visibility:list=None): + host:str="github.com", forge:str="", subdir:str="", visibility:list=None): """Downloads a crate's source from a git forge archive (for forks and unpublished revisions), normalized to the {crate}-{version} layout that rust_repo expects. The go-rules `download` pattern: the crate keeps its @@ -1254,15 +1300,17 @@ def rust_git_download(name:str, crate:str, version:str, repo:str, revision:str, name: Name of the rule. crate: Crate name (canonical, as dependents refer to it). version: Version dependents resolve against. - repo: owner/repo on the forge. + repo: `owner/repo` on github, or a full repository URL on any forge. revision: Commit sha or tag to fetch. hashes: Optional archive hashes to verify. - host: Forge host serving /archive/ tarballs (github.com, gitlab works too). + host: Forge host, when repo is the shorthand. + forge: Archive scheme to use, 'github' or 'gitlab'. Inferred from the + host when empty. subdir: Path of the crate within the repo, for workspace members. """ dl = remote_file( name = tag(name, "git"), - url = f"https://{host}/{repo}/archive/{revision}.tar.gz", + url = forge_archive_url(repo, revision, host, forge), hashes = hashes, extract = True, ) @@ -1333,6 +1381,7 @@ def rust_repo( download: str = None, git_repo: str = "", git_revision: str = "", + git_forge: str = "", platforms: list = None, ): """Downloads a crate and generates BUILD files as a subrepo. @@ -1378,9 +1427,13 @@ def rust_repo( download: A rule providing the crate source (a {crate}-{version} dir with Cargo.toml at its root) instead of the crates.io tarball — e.g. a rust_git_download of a fork. - git_repo: owner/repo shorthand: fetch the source from this git forge - repository instead of crates.io. + git_repo: `owner/repo` on github, or a full repository URL on any + forge, to fetch the source from rather than crates.io. git_revision: Commit sha or tag for git_repo. + git_forge: Archive scheme the forge serves, 'github' or 'gitlab'. + Inferred from the host when empty, which is right for + gitlab.com and wrong for gitlab self-hosted somewhere its + name does not say. platforms: The operating systems (in Rust's `target_os` vocabulary) whose resolution reaches this crate. Omitted for the overwhelming majority of crates, which build anywhere; @@ -1436,6 +1489,7 @@ def rust_repo( version = version, repo = git_repo, revision = git_revision or version, + forge = git_forge, hashes = hashes, ) elif not download: diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index fab5990..e69c033 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -47,7 +47,6 @@ rather than by release: [#15](https://github.com/becomeliminal/rust-rules/issues/15) remote execution audit, [#16](https://github.com/becomeliminal/rust-rules/issues/16) musl and embedded - **Missing capability:** - [#21](https://github.com/becomeliminal/rust-rules/issues/21) git forges, [#23](https://github.com/becomeliminal/rust-rules/issues/23) cross-compiling C, [#24](https://github.com/becomeliminal/rust-rules/issues/24) channels, [#25](https://github.com/becomeliminal/rust-rules/issues/25) cbindgen, @@ -82,7 +81,7 @@ rather than by release: | **Add a dependency in one command**
No repin step | **yes**. lock --add crate@req, re-solves and declares | **partial**. cargo add then a repin of crate_universe | **yes**. cargo add | | **Upgrade to the newest compatible versions**
The cargo update equivalent | **yes**. lock --upgrade for everything, or named crates; stays inside each declaration's compatibility range | **no**. Re-run the generator against new requirements | **yes**. cargo update | | **Import an existing Cargo.lock**
Adopting a repo that already uses cargo | **yes**. sync --import, and --import-workspace for BUILD files too | **yes**. crate_universe consumes Cargo.toml directly | **yes**. Native | -| **Git and fork dependencies**
Pinned revision instead of crates.io | **partial**. github archive URLs; other forges need download= | **yes**. Supported | **yes**. Native | +| **Git and fork dependencies**
Pinned revision instead of crates.io | **yes**. Any forge serving source archives, github and gitlab schemes both | **yes**. Supported | **yes**. Native | | **Private or alternative registries**
Registry auth | **no**. On demand only; forks and download= cover the cases | **yes**. Via cargo | **yes**. Native | | **Vendored source overrides**
Patch or replace a crate's source | **yes**. download= overrides the source, patch= applies patches to it | **yes**. annotations and patches | **yes**. [patch] and [replace] | diff --git a/test/forge/BUILD b/test/forge/BUILD new file mode 100644 index 0000000..e6d7b2a --- /dev/null +++ b/test/forge/BUILD @@ -0,0 +1,50 @@ +subinclude("//build_defs:rust") + +# Where a crate's source comes from when it is not crates.io. +# +# Most forges serve /archive/.tar.gz: github, gitea and its forks such +# as codeberg, and sourcehut all do. GitLab does not. Its /archive/ path +# answers 403 and the tarball lives at /-/archive//-.tar.gz, +# so a gitlab repo declared as though it were github fails at download with +# nothing to say why. +# +# These run when the package is parsed, so a change to the URL scheme fails +# the build rather than a fetch. +_forge_cases = [ + # repo, forge, expected URL + ["owner/repo", "", "https://github.com/owner/repo/archive/abc123.tar.gz"], + [ + "https://github.com/owner/repo", + "", + "https://github.com/owner/repo/archive/abc123.tar.gz", + ], + # A trailing .git is how a Cargo.lock spells the same repository. + [ + "https://codeberg.org/owner/repo.git", + "", + "https://codeberg.org/owner/repo/archive/abc123.tar.gz", + ], + # gitlab.com is recognised by its host, and the tarball is named for the + # project rather than the whole path, so a repo inside a group nests. + [ + "https://gitlab.com/group/sub/proj", + "", + "https://gitlab.com/group/sub/proj/-/archive/abc123/proj-abc123.tar.gz", + ], + # A forge whose name says nothing about what it runs has to be told. + [ + "https://git.corp.example/team/proj", + "gitlab", + "https://git.corp.example/team/proj/-/archive/abc123/proj-abc123.tar.gz", + ], + [ + "https://git.corp.example/team/proj", + "", + "https://git.corp.example/team/proj/archive/abc123.tar.gz", + ], +] + +for _case in _forge_cases: + _got = forge_archive_url(_case[0], "abc123", "github.com", _case[1]) + if _got != _case[2]: + fail("forge_archive_url(%s, forge=%s) = %s, want %s" % (_case[0], _case[1], _got, _case[2])) diff --git a/tools/please_rust/src/sync.rs b/tools/please_rust/src/sync.rs index 2518e82..5ce7835 100644 --- a/tools/please_rust/src/sync.rs +++ b/tools/please_rust/src/sync.rs @@ -780,15 +780,17 @@ fn import_cargo_lock(path: &Path, decls: &mut Vec) -> Result<()> { let (url, frag) = rest.split_once('#').unwrap_or((rest, "")); let url = url.split('?').next().unwrap_or(url); if let Some(path) = url.strip_prefix("https://github.com/") { + // The shorthand, because that is what the rule has always + // recorded for github and what its declarations look like. git_repo = path.trim_end_matches(".git").to_string(); - git_revision = frag.to_string(); } else { - eprintln!( - "warning: {} uses a non-github git source ({}); declare it manually with rust_repo(download = ...)", - name, url - ); - continue; + // Any other forge is recorded as the URL it was cloned from. + // The rule derives the archive scheme from the host, which is + // right except for gitlab hosted somewhere its name does not + // say; those need git_forge = "gitlab" adding by hand. + git_repo = url.trim_end_matches(".git").to_string(); } + git_revision = frag.to_string(); if git_revision.is_empty() { eprintln!( "warning: {} git source has no pinned revision, skipping", @@ -2303,8 +2305,13 @@ fresh = { version = "2", features = ["extra"] } let forked = decls.iter().find(|d| d.crate_name == "forked").unwrap(); assert_eq!(forked.git_repo, "owner/forked"); assert_eq!(forked.git_revision, "abcdef123456"); - // Non-github git and local path crates skipped - assert!(!decls.iter().any(|d| d.crate_name == "elsewhere")); + // A git source on any other forge is imported as the URL it was + // cloned from, rather than skipped with a note to write the rule by + // hand. The rule derives the archive scheme from the host. + let elsewhere = decls.iter().find(|d| d.crate_name == "elsewhere").unwrap(); + assert_eq!(elsewhere.git_repo, "https://gitlab.example.com/x/y"); + assert_eq!(elsewhere.git_revision, "deadbeef"); + // A path dependency has no source to fetch, so it stays skipped. assert!(!decls.iter().any(|d| d.crate_name == "local_thing")); } From 8301961dc7fdfc7be095efca4cbae8435643ad7f Mon Sep 17 00:00:00 2001 From: Jack Marsh Date: Mon, 24 Aug 2026 12:01:19 +0100 Subject: [PATCH 2/2] Escape a slash in a gitlab tag On gitlab the revision is its own path segment followed by the archive name, so a tag containing a slash has to be escaped or gitlab reads the part before the slash as the whole ref. sequoia tags releases openpgp/v2.4.1 because one repository holds several crates. Unescaped it resolves today, but only because no branch is named openpgp; one appearing would silently fetch a different tree. The escaped URL is verified against gitlab and returns the same 9,468,234 bytes. Github takes the ref as the last segment before the extension, where a slash needs no escaping and escaping it would break the URL, so this applies to the gitlab scheme only. --- build_defs/rust.build_defs | 14 ++++++++++++-- test/forge/BUILD | 13 +++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/build_defs/rust.build_defs b/build_defs/rust.build_defs index b265b6b..066e759 100644 --- a/build_defs/rust.build_defs +++ b/build_defs/rust.build_defs @@ -1283,9 +1283,19 @@ def forge_archive_url(repo:str, revision:str, host:str="github.com", forge:str=" kind = "gitlab" if forge_host == "gitlab.com" or forge_host.startswith("gitlab.") else "github" if kind == "gitlab": # The tarball is named for the project, which is the last path - # segment, not for the whole path: a repo under a group nests. + # segment, not for the whole path: a repo under a group nests. The + # name itself is cosmetic, gitlab serves the archive whatever it says. project = path.split("/")[-1] - return f"{base}/-/archive/{revision}/{project}-{revision}.tar.gz" + # The revision is its own path segment here, followed by that name, so + # a tag with a slash in it has to be escaped or gitlab reads the part + # before the slash as the whole ref. sequoia tags releases + # openpgp/v2.4.1 because one repository holds several crates. It + # happens to resolve unescaped today only because no branch is named + # openpgp; one appearing would silently fetch a different tree. + ref = revision.replace("/", "%2F") + return f"{base}/-/archive/{ref}/{project}-{revision}.tar.gz" + # Github takes the ref as the last segment before the extension, where a + # slash needs no escaping and escaping it would break the URL. return f"{base}/archive/{revision}.tar.gz" diff --git a/test/forge/BUILD b/test/forge/BUILD index e6d7b2a..9899eee 100644 --- a/test/forge/BUILD +++ b/test/forge/BUILD @@ -44,6 +44,19 @@ _forge_cases = [ ], ] +# A tag can contain a slash. On gitlab the revision is its own path segment, +# so it has to be escaped: sequoia tags releases openpgp/v2.4.1 because one +# repository holds several crates. +_slash = forge_archive_url("https://gitlab.com/sequoia-pgp/sequoia", "openpgp/v2.4.1", "github.com", "") +_want_slash = "https://gitlab.com/sequoia-pgp/sequoia/-/archive/openpgp%2Fv2.4.1/sequoia-openpgp/v2.4.1.tar.gz" +if _slash != _want_slash: + fail("slashed gitlab tag: got %s, want %s" % (_slash, _want_slash)) + +# Github takes the ref as the last segment, where a slash needs no escaping. +_ghslash = forge_archive_url("owner/repo", "release/1.0", "github.com", "") +if _ghslash != "https://github.com/owner/repo/archive/release/1.0.tar.gz": + fail("slashed github ref should not be escaped: %s" % _ghslash) + for _case in _forge_cases: _got = forge_archive_url(_case[0], "abc123", "github.com", _case[1]) if _got != _case[2]: