Fetch a crate from any forge, not only github

README.md

@@ -179,8 +179,24 @@ rust_repo(
     git_revision = "1.0.86",
 )
 ```
-(`sync --import` translates `git+https://github.com/...` lockfile sources
-automatically.)
+`git_repo` also takes a full URL, for any forge:
+```python
+rust_repo(
+    name = "thing",
+    crate = "thing",
+    version = "0.3.0",
+    git_repo = "https://gitlab.com/group/thing",
+    git_revision = "v0.3.0",
+)
+```
+Most forges serve `/archive/<rev>.tar.gz`, which is what github, gitea and
+its forks, and sourcehut all do. GitLab serves
+`/-/archive/<rev>/<project>-<rev>.tar.gz`, and is recognised by its host.
+GitLab running somewhere its name does not say needs `git_forge = "gitlab"`.
+A forge serving neither scheme needs `download = ...`, naming any rule that
+produces the crate's source.
+
+`sync --import` translates `git+https://` lockfile sources from any host.
 
 `rust_library` builds an `rlib` by default; `crate_type` also supports
 `proc-macro`, `dylib`, `cdylib` and `staticlib` for compiler plugins and

build_defs/rust.build_defs

@@ -1243,8 +1243,64 @@ def rust_crate_download(name:str, crate:str, version:str, hashes:list=None, labe
     )
 
 
+def forge_archive_url(repo:str, revision:str, host:str="github.com", forge:str=""):
+    """The URL of a source archive for one revision of a repository.
+
+    Most forges serve `/archive/<rev>.tar.gz`: github, gitea and its forks
+    (codeberg), and sourcehut all do. GitLab is the exception and serves
+    `/-/archive/<rev>/<project>-<rev>.tar.gz`; its `/archive/` path answers
+    403, so a gitlab repo declared as though it were github fails at download
+    with no hint as to why.
+
+    The host decides the scheme, which cannot work for a self-hosted forge
+    whose name says nothing about what it runs. `forge` says it outright.
+
+    Args:
+        repo: Either `owner/repo`, which means github.com, or a full URL to
+              the repository on any forge.
+        revision: Commit sha or tag to fetch.
+        host: Forge host, when repo is the shorthand.
+        forge: 'github' or 'gitlab' to say which archive scheme the forge
+               serves. Inferred from the host when empty, which is right for
+               gitlab.com and wrong for gitlab running somewhere else.
+    """
+    if repo.startswith("https://") or repo.startswith("http://"):
+        base = repo.rstrip("/")
+        if base.endswith(".git"):
+            base = base[:-4]
+        # Split on the whole path rather than with a maxsplit, which this
+        # dialect does not take.
+        rest = base.replace("https://", "").replace("http://", "")
+        segments = rest.split("/")
+        forge_host = segments[0]
+        path = "/".join(segments[1:])
+    else:
+        forge_host = host
+        path = repo.strip("/")
+        base = f"https://{forge_host}/{path}"
+    kind = forge
+    if not kind:
+        kind = "gitlab" if forge_host == "gitlab.com" or forge_host.startswith("gitlab.") else "github"
+    if kind == "gitlab":
+        # The tarball is named for the project, which is the last path
+        # segment, not for the whole path: a repo under a group nests. The
+        # name itself is cosmetic, gitlab serves the archive whatever it says.
+        project = path.split("/")[-1]
+        # The revision is its own path segment here, followed by that name, so
+        # a tag with a slash in it has to be escaped or gitlab reads the part
+        # before the slash as the whole ref. sequoia tags releases
+        # openpgp/v2.4.1 because one repository holds several crates. It
+        # happens to resolve unescaped today only because no branch is named
+        # openpgp; one appearing would silently fetch a different tree.
+        ref = revision.replace("/", "%2F")
+        return f"{base}/-/archive/{ref}/{project}-{revision}.tar.gz"
+    # Github takes the ref as the last segment before the extension, where a
+    # slash needs no escaping and escaping it would break the URL.
+    return f"{base}/archive/{revision}.tar.gz"
+
+
 def rust_git_download(name:str, crate:str, version:str, repo:str, revision:str, hashes:list=None,
-                      host:str="github.com", subdir:str="", visibility:list=None):
+                      host:str="github.com", forge:str="", subdir:str="", visibility:list=None):
     """Downloads a crate's source from a git forge archive (for forks and
     unpublished revisions), normalized to the {crate}-{version} layout that
     rust_repo expects. The go-rules `download` pattern: the crate keeps its
@@ -1254,15 +1310,17 @@ def rust_git_download(name:str, crate:str, version:str, repo:str, revision:str,
         name: Name of the rule.
         crate: Crate name (canonical, as dependents refer to it).
         version: Version dependents resolve against.
-        repo: owner/repo on the forge.
+        repo: `owner/repo` on github, or a full repository URL on any forge.
         revision: Commit sha or tag to fetch.
         hashes: Optional archive hashes to verify.
-        host: Forge host serving /archive/ tarballs (github.com, gitlab works too).
+        host: Forge host, when repo is the shorthand.
+        forge: Archive scheme to use, 'github' or 'gitlab'. Inferred from the
+               host when empty.
         subdir: Path of the crate within the repo, for workspace members.
     """
     dl = remote_file(
         name = tag(name, "git"),
-        url = f"https://{host}/{repo}/archive/{revision}.tar.gz",
+        url = forge_archive_url(repo, revision, host, forge),
         hashes = hashes,
         extract = True,
     )
@@ -1333,6 +1391,7 @@ def rust_repo(
     download: str = None,
     git_repo: str = "",
     git_revision: str = "",
+    git_forge: str = "",
     platforms: list = None,
 ):
     """Downloads a crate and generates BUILD files as a subrepo.
@@ -1378,9 +1437,13 @@ def rust_repo(
         download: A rule providing the crate source (a {crate}-{version} dir
                   with Cargo.toml at its root) instead of the crates.io
                   tarball — e.g. a rust_git_download of a fork.
-        git_repo: owner/repo shorthand: fetch the source from this git forge
-                  repository instead of crates.io.
+        git_repo: `owner/repo` on github, or a full repository URL on any
+                  forge, to fetch the source from rather than crates.io.
         git_revision: Commit sha or tag for git_repo.
+        git_forge: Archive scheme the forge serves, 'github' or 'gitlab'.
+                   Inferred from the host when empty, which is right for
+                   gitlab.com and wrong for gitlab self-hosted somewhere its
+                   name does not say.
         platforms: The operating systems (in Rust's `target_os` vocabulary)
                    whose resolution reaches this crate. Omitted for the
                    overwhelming majority of crates, which build anywhere;
@@ -1436,6 +1499,7 @@ def rust_repo(
             version = version,
             repo = git_repo,
             revision = git_revision or version,
+            forge = git_forge,
             hashes = hashes,
         )
     elif not download:

docs/COMPARISON.md

@@ -47,7 +47,6 @@ rather than by release:
   [#15](https://github.com/becomeliminal/rust-rules/issues/15) remote execution audit,
   [#16](https://github.com/becomeliminal/rust-rules/issues/16) musl and embedded
 - **Missing capability:**
-  [#21](https://github.com/becomeliminal/rust-rules/issues/21) git forges,
   [#23](https://github.com/becomeliminal/rust-rules/issues/23) cross-compiling C,
   [#24](https://github.com/becomeliminal/rust-rules/issues/24) channels,
   [#25](https://github.com/becomeliminal/rust-rules/issues/25) cbindgen,
@@ -82,7 +81,7 @@ rather than by release:
 | **Add a dependency in one command**<br>No repin step | **yes**. lock --add crate@req, re-solves and declares | **partial**. cargo add then a repin of crate_universe | **yes**. cargo add |
 | **Upgrade to the newest compatible versions**<br>The cargo update equivalent | **yes**. lock --upgrade for everything, or named crates; stays inside each declaration's compatibility range | **no**. Re-run the generator against new requirements | **yes**. cargo update |
 | **Import an existing Cargo.lock**<br>Adopting a repo that already uses cargo | **yes**. sync --import, and --import-workspace for BUILD files too | **yes**. crate_universe consumes Cargo.toml directly | **yes**. Native |
-| **Git and fork dependencies**<br>Pinned revision instead of crates.io | **partial**. github archive URLs; other forges need download= | **yes**. Supported | **yes**. Native |
+| **Git and fork dependencies**<br>Pinned revision instead of crates.io | **yes**. Any forge serving source archives, github and gitlab schemes both | **yes**. Supported | **yes**. Native |
 | **Private or alternative registries**<br>Registry auth | **no**. On demand only; forks and download= cover the cases | **yes**. Via cargo | **yes**. Native |
 | **Vendored source overrides**<br>Patch or replace a crate's source | **yes**. download= overrides the source, patch= applies patches to it | **yes**. annotations and patches | **yes**. [patch] and [replace] |
 

test/forge/BUILD

@@ -0,0 +1,63 @@
+subinclude("//build_defs:rust")
+
+# Where a crate's source comes from when it is not crates.io.
+#
+# Most forges serve /archive/<rev>.tar.gz: github, gitea and its forks such
+# as codeberg, and sourcehut all do. GitLab does not. Its /archive/ path
+# answers 403 and the tarball lives at /-/archive/<rev>/<project>-<rev>.tar.gz,
+# so a gitlab repo declared as though it were github fails at download with
+# nothing to say why.
+#
+# These run when the package is parsed, so a change to the URL scheme fails
+# the build rather than a fetch.
+_forge_cases = [
+    # repo, forge, expected URL
+    ["owner/repo", "", "https://github.com/owner/repo/archive/abc123.tar.gz"],
+    [
+        "https://github.com/owner/repo",
+        "",
+        "https://github.com/owner/repo/archive/abc123.tar.gz",
+    ],
+    # A trailing .git is how a Cargo.lock spells the same repository.
+    [
+        "https://codeberg.org/owner/repo.git",
+        "",
+        "https://codeberg.org/owner/repo/archive/abc123.tar.gz",
+    ],
+    # gitlab.com is recognised by its host, and the tarball is named for the
+    # project rather than the whole path, so a repo inside a group nests.
+    [
+        "https://gitlab.com/group/sub/proj",
+        "",
+        "https://gitlab.com/group/sub/proj/-/archive/abc123/proj-abc123.tar.gz",
+    ],
+    # A forge whose name says nothing about what it runs has to be told.
+    [
+        "https://git.corp.example/team/proj",
+        "gitlab",
+        "https://git.corp.example/team/proj/-/archive/abc123/proj-abc123.tar.gz",
+    ],
+    [
+        "https://git.corp.example/team/proj",
+        "",
+        "https://git.corp.example/team/proj/archive/abc123.tar.gz",
+    ],
+]
+
+# A tag can contain a slash. On gitlab the revision is its own path segment,
+# so it has to be escaped: sequoia tags releases openpgp/v2.4.1 because one
+# repository holds several crates.
+_slash = forge_archive_url("https://gitlab.com/sequoia-pgp/sequoia", "openpgp/v2.4.1", "github.com", "")
+_want_slash = "https://gitlab.com/sequoia-pgp/sequoia/-/archive/openpgp%2Fv2.4.1/sequoia-openpgp/v2.4.1.tar.gz"
+if _slash != _want_slash:
+    fail("slashed gitlab tag: got %s, want %s" % (_slash, _want_slash))
+
+# Github takes the ref as the last segment, where a slash needs no escaping.
+_ghslash = forge_archive_url("owner/repo", "release/1.0", "github.com", "")
+if _ghslash != "https://github.com/owner/repo/archive/release/1.0.tar.gz":
+    fail("slashed github ref should not be escaped: %s" % _ghslash)
+
+for _case in _forge_cases:
+    _got = forge_archive_url(_case[0], "abc123", "github.com", _case[1])
+    if _got != _case[2]:
+        fail("forge_archive_url(%s, forge=%s) = %s, want %s" % (_case[0], _case[1], _got, _case[2]))

tools/please_rust/src/sync.rs

@@ -780,15 +780,17 @@ fn import_cargo_lock(path: &Path, decls: &mut Vec<Decl>) -> Result<()> {
             let (url, frag) = rest.split_once('#').unwrap_or((rest, ""));
             let url = url.split('?').next().unwrap_or(url);
             if let Some(path) = url.strip_prefix("https://github.com/") {
+                // The shorthand, because that is what the rule has always
+                // recorded for github and what its declarations look like.
                 git_repo = path.trim_end_matches(".git").to_string();
-                git_revision = frag.to_string();
             } else {
-                eprintln!(
-                    "warning: {} uses a non-github git source ({}); declare it manually with rust_repo(download = ...)",
-                    name, url
-                );
-                continue;
+                // Any other forge is recorded as the URL it was cloned from.
+                // The rule derives the archive scheme from the host, which is
+                // right except for gitlab hosted somewhere its name does not
+                // say; those need git_forge = "gitlab" adding by hand.
+                git_repo = url.trim_end_matches(".git").to_string();
             }
+            git_revision = frag.to_string();
             if git_revision.is_empty() {
                 eprintln!(
                     "warning: {} git source has no pinned revision, skipping",
@@ -2303,8 +2305,13 @@ fresh = { version = "2", features = ["extra"] }
         let forked = decls.iter().find(|d| d.crate_name == "forked").unwrap();
         assert_eq!(forked.git_repo, "owner/forked");
         assert_eq!(forked.git_revision, "abcdef123456");
-        // Non-github git and local path crates skipped
-        assert!(!decls.iter().any(|d| d.crate_name == "elsewhere"));
+        // A git source on any other forge is imported as the URL it was
+        // cloned from, rather than skipped with a note to write the rule by
+        // hand. The rule derives the archive scheme from the host.
+        let elsewhere = decls.iter().find(|d| d.crate_name == "elsewhere").unwrap();
+        assert_eq!(elsewhere.git_repo, "https://gitlab.example.com/x/y");
+        assert_eq!(elsewhere.git_revision, "deadbeef");
+        // A path dependency has no source to fetch, so it stays skipped.
         assert!(!decls.iter().any(|d| d.crate_name == "local_thing"));
     }