Link LLD's COFF, ELF and Mach-O linkers into talac

.github/workflows/compiler-tier1.yml

@@ -202,6 +202,44 @@ jobs:
           llvm_root="$(brew --prefix llvm)"
           echo "LLVM_ROOT=$llvm_root" >> "$GITHUB_ENV"
           echo "$llvm_root/bin" >> "$GITHUB_PATH"
+          echo "LLVM_VERSION=$("$llvm_root/bin/llvm-config" --version)" >> "$GITHUB_ENV"
+
+      # talac links LLD's linkers statically. Homebrew's lld formula has them
+      # only as shared libraries over libLLVM.dylib, which would put a second
+      # LLVM into talac, so their static libraries are built here from the
+      # sources of the same LLVM version, against Homebrew's LLVM, as that
+      # formula builds them; the result is kept per version.
+      - name: Restore LLD's static libraries
+        id: lld-cache
+        uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: ${{ runner.temp }}/lld-static
+          key: lld-static-${{ runner.os }}-${{ runner.arch }}-${{ env.LLVM_VERSION }}
+
+      - name: Build LLD's static libraries
+        if: steps.lld-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          source="llvm-project-${LLVM_VERSION}.src"
+          curl -fsSLo llvm-project.tar.xz \
+            "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${source}.tar.xz"
+          tar -xJf llvm-project.tar.xz "$source/lld" "$source/libunwind/include" "$source/cmake" "$source/llvm/utils/lit"
+          cmake -G Ninja -S "$source/lld" -B lld-build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX="$RUNNER_TEMP/lld-static" \
+            -DLLVM_DIR="$LLVM_ROOT/lib/cmake/llvm" \
+            -DBUILD_SHARED_LIBS=OFF \
+            -DLLVM_INCLUDE_TESTS=OFF
+          cmake --build lld-build --target lldCommon lldCOFF lldELF lldMachO
+          mkdir -p "$RUNNER_TEMP/lld-static/lib" "$RUNNER_TEMP/lld-static/include"
+          cp lld-build/lib/liblldCommon.a lld-build/lib/liblldCOFF.a \
+            lld-build/lib/liblldELF.a lld-build/lib/liblldMachO.a "$RUNNER_TEMP/lld-static/lib/"
+          cp -R "$source/lld/include/lld" "$RUNNER_TEMP/lld-static/include/"
+          cp -R lld-build/include/lld/. "$RUNNER_TEMP/lld-static/include/lld/" 2>/dev/null || true
+
+      - name: Use LLD's static libraries
+        shell: bash
+        run: echo "LLD_ROOT=$RUNNER_TEMP/lld-static" >> "$GITHUB_ENV"
 
       - name: Install GHC and Cabal
         uses: haskell/ghcup-setup@269e121e9e7528761aca5a606a53fe9d04cd6658 # v1.2.3

CHANGELOG.md

@@ -9,11 +9,22 @@ SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
 
 ### Linking
 
-- An executable is linked for its target rather than for the host: Windows
-  executables with `lld-link` from any host, for the machine of the target;
-  another Linux, macOS or BSD target with `clang --target=<triple>
-  -fuse-ld=lld`; the host's own target with `cc` as before. The runtime of
-  another target is looked up in `runtimes/<triple>/` beside talac.
+- LLD's COFF, ELF and Mach-O linkers are linked into talac, which links
+  executables itself. A Windows executable needs nothing from the host, and
+  its import libraries are written by LLVM. For Linux, macOS and the BSDs
+  talac takes the command line of the linker from `clang -###`, which knows
+  where the target's C library is, and runs it with LLD; without `clang`, a
+  target of the host is linked by `cc` as before. Building talac needs LLD's
+  static libraries and headers: the LLVM release for Windows has them, the
+  Linux toolchain carries them from revision 2, and on macOS they are built
+  from the LLVM sources against Homebrew's LLVM, because Homebrew's `lld` has
+  only shared libraries; `LLD_ROOT` names them where they are not beside
+  LLVM.
+
+- An executable is linked for its target rather than for the host: a Windows
+  executable for the machine of the target from any host, and another Linux,
+  macOS or BSD target through `clang --target=<triple>`. The runtime of another
+  target is looked up in `runtimes/<triple>/` beside talac.
 
 - `--sysroot=<path>` names the system root the linker driver links against
   on Linux, macOS and the BSDs; it must be a directory. A Windows target

Documents/CLI.md

@@ -83,13 +83,19 @@ Several sources are one program, and a program is lowered from its entry, so a c
 
 ### Linking for another target
 
-The target, not the host, decides how an executable is linked:
+Talac links executables itself: LLD's COFF, ELF and Mach-O linkers are linked into it. The target, not the host, decides
+how:
 
 | Target | Linker | TalaRT, shared | TalaRT, archive |
 | --- | --- | --- | --- |
-| Windows (`x86_64`, `aarch64`) | `lld-link`, on every host | `talart.dll` | `talart.lib` |
-| the host's own system, architecture and C library | `cc`, the C compiler of the host | `libtalart.so`, `libtalart.dylib` | `libtalart.a` |
-| another Linux, macOS or BSD target | `clang --target=<triple> -fuse-ld=lld` | `libtalart.so`, `libtalart.dylib` | `libtalart.a` |
+| Windows (`x86_64`, `aarch64`) | LLD's `lld-link`, on every host, with nothing from the host | `talart.dll` | `talart.lib` |
+| the host's own system, architecture and C library | LLD, with the command line `clang` gives for its linker | `libtalart.so`, `libtalart.dylib` | `libtalart.a` |
+| another Linux, macOS or BSD target | LLD, with the command line `clang --target=<triple> -fuse-ld=lld` gives | `libtalart.so`, `libtalart.dylib` | `libtalart.a` |
+
+A Linux, macOS or BSD executable needs the start-up objects and the C library of its target, whose places a C compiler
+knows. Talac asks `clang` for the command line of its linker (`-###`), which runs nothing, and runs that command line with
+LLD. When no `clang` is on `PATH`, a target of the host is linked by `cc` and its own linker instead. Import libraries for
+Windows are written by talac too.
 
 TalaRT of the host stands beside talac. TalaRT of another target stands in `runtimes/<triple>/` beside it, under the
 normalized triple, for example `runtimes/aarch64-unknown-linux-gnu/libtalart.so`; a program that calls nothing of TalaRT
@@ -101,7 +107,7 @@ talac Program.tala --entry=Demo.Program --target=x86_64-pc-windows-msvc
 talac Program.tala --entry=Demo.Program --target=aarch64-unknown-linux-gnu --sysroot=/opt/sysroots/arm64
 ```
 
-`lld-link`, `cc` and `clang` are found on `PATH`.
+`clang`, and `cc` where no `clang` is found, are looked up on `PATH`; no linker is.
 
 ### TalaRT
 

MODULE.bazel

@@ -10,6 +10,12 @@ bazel_dep(name = "catch3", version = "0.2.0")
 bazel_dep(name = "google_benchmark", version = "1.9.5")
 bazel_dep(name = "indicators", version = "2.3.0")
 bazel_dep(name = "tabulate", version = "1.5.0")
+# LLVM's release for Windows was built with zlib and zstd but does not ship
+# them, nor libxml2; LLD, which talac links in, compresses with zlib and
+# zstd and merges Windows manifests with libxml2.
+bazel_dep(name = "libxml2", version = "2.15.3")
+bazel_dep(name = "zlib", version = "1.3.2.bcr.3")
+bazel_dep(name = "zstd", version = "1.5.7.bcr.3")
 
 # Neither header-only CLI library is currently published in the Bazel Central
 # Registry. Their upstream repositories are pinned as submodules beneath a

Talac/Build/Bazel/llvm.bzl

@@ -1,4 +1,28 @@
-"""Discovers an installed LLVM development tree for the C++20 backend."""
+"""Discovers an installed LLVM development tree for the C++20 backend and LLD."""
+
+# The components of LLVM that LLD's COFF, ELF and Mach-O linkers use, beside
+# those of the backend. They are linked into talac with LLD.
+_LLD_COMPONENTS = [
+    # LLD initializes every target LLVM was built for, for link-time
+    # optimization.
+    "all-targets",
+    "debuginfocodeview",
+    "debuginfodwarf",
+    "debuginfomsf",
+    "debuginfopdb",
+    "demangle",
+    "libdriver",
+    "lto",
+    "objcarcopts",
+    "option",
+    "textapi",
+    "windowsdriver",
+    "windowsmanifest",
+]
+
+# LLD's libraries, in the order a static link needs them: the linkers before
+# what they share.
+_LLD_LIBRARIES = ["lldCOFF", "lldELF", "lldMachO", "lldCommon"]
 
 _COMPONENTS = [
     "core",
@@ -101,8 +125,15 @@ def _llvm_repository_impl(repository_ctx):
             library_dir,
         ))
 
+    # Distributed ThinLTO is a component of its own since LLVM 21; LLD uses
+    # it where LLVM has it.
+    available = repository_ctx.execute([config, "--components"], quiet = True)
+    lld_components = list(_LLD_COMPONENTS)
+    if available.return_code == 0 and "dtlto" in available.stdout.split(" "):
+        lld_components.append("dtlto")
+
     libraries = repository_ctx.execute(
-        [config, "--link-static", "--libnames"] + _COMPONENTS,
+        [config, "--link-static", "--libnames"] + _COMPONENTS + lld_components,
         quiet = True,
     )
     if libraries.return_code != 0:
@@ -132,7 +163,7 @@ def _llvm_repository_impl(repository_ctx):
         ]
     else:
         system_libraries = repository_ctx.execute(
-            [config, "--link-static", "--system-libs"] + _COMPONENTS,
+            [config, "--link-static", "--system-libs"] + _COMPONENTS + lld_components,
             quiet = True,
         )
         if system_libraries.return_code != 0:
@@ -162,6 +193,8 @@ def _llvm_repository_impl(repository_ctx):
     repository_ctx.symlink(include_dir, "include")
     repository_ctx.symlink(library_dir, "lib")
 
+    lld_imports = _lld_imports(repository_ctx, include_dir, library_dir, library_suffix, is_windows)
+
     support_archive = "lib/LLVMSupport.lib"
     if is_windows and repository_ctx.os.environ.get("TALA_LLVM_SYSTEM_ALLOCATOR") == "1":
         support_archive = _sanitizer_support_archive(repository_ctx, config, library_dir, library_names)
@@ -194,12 +227,76 @@ cc_library(
     deps = {},
     linkopts = {},
 )
-""".format("\n".join(imports), repr(dependencies), repr(system_linkopts))
+
+{}
+""".format("\n".join(imports), repr(dependencies), repr(system_linkopts), lld_imports)
     repository_ctx.file("BUILD.bazel", build)
 
+def _lld_imports(repository_ctx, include_dir, library_dir, library_suffix, is_windows):
+    """The BUILD text of the `lld` target: LLD's linkers and their headers.
+
+    The Windows release of LLVM and the libc++ toolchain of the Linux hosts
+    keep LLD beside LLVM. Homebrew keeps it in a formula of its own, which
+    LLD_ROOT names or `brew --prefix lld` finds.
+    """
+    prefix = "" if is_windows else "lib"
+
+    def has_lld(include, library):
+        return include.get_child("lld").get_child("Common").get_child("Driver.h").exists and all([
+            library.get_child("{}{}{}".format(prefix, name, library_suffix)).exists
+            for name in _LLD_LIBRARIES
+        ])
+
+    lld_include = include_dir
+    lld_library = library_dir
+    if not has_lld(lld_include, lld_library):
+        root = repository_ctx.os.environ.get("LLD_ROOT")
+        if not root:
+            brew = repository_ctx.which("brew")
+            if brew:
+                found = repository_ctx.execute([brew, "--prefix", "lld"], quiet = True)
+                if found.return_code == 0:
+                    root = found.stdout.strip()
+        if root:
+            lld_include = repository_ctx.path(root).get_child("include")
+            lld_library = repository_ctx.path(root).get_child("lib")
+        if not root or not has_lld(lld_include, lld_library):
+            fail("LLD's static libraries and headers were not found beside LLVM. Set LLD_ROOT to a tree " +
+                 "with lib/ and include/lld built for this LLVM; Homebrew's lld formula has only shared libraries.")
+        repository_ctx.symlink(lld_include, "lld-include")
+        repository_ctx.symlink(lld_library, "lld-lib")
+        header_root = "lld-include"
+        library_root = "lld-lib"
+    else:
+        header_root = "include"
+        library_root = "lib"
+
+    text = ["""cc_library(
+    name = \"lld_headers\",
+    hdrs = glob([\"{root}/lld/**/*.h\"]),
+    includes = [\"{root}\"],
+)""".format(root = header_root)]
+    previous = ":llvm"
+    for name in reversed(_LLD_LIBRARIES):
+        text.append("""cc_import(
+    name = {target},
+    static_library = {library},
+    deps = [{previous}, \":lld_headers\"],
+)""".format(
+            target = _quote(name),
+            library = _quote("{}/{}{}{}".format(library_root, prefix, name, library_suffix)),
+            previous = _quote(previous),
+        ))
+        previous = ":" + name
+    text.append("""cc_library(
+    name = \"lld\",
+    deps = [{}],
+)""".format(", ".join([_quote(":" + name) for name in _LLD_LIBRARIES[:-1]])))
+    return "\n\n".join(text)
+
 _llvm_repository = repository_rule(
     implementation = _llvm_repository_impl,
-    environ = ["LLVM_ROOT", "PATH", "TALA_LLVM_SYSTEM_ALLOCATOR"],
+    environ = ["LLD_ROOT", "LLVM_ROOT", "PATH", "TALA_LLVM_SYSTEM_ALLOCATOR"],
     local = True,
 )
 

Talac/Linker/BUILD.bazel

@@ -14,12 +14,35 @@ cc_library(
     ],
 )
 
+# LLD's COFF, ELF and Mach-O linkers, linked into talac so that it links
+# executables itself.
+cc_library(
+    name = "lld",
+    srcs = ["Lld.cpp"],
+    hdrs = ["Lld.hpp"],
+    deps = [
+        "@llvm//:lld",
+        "@llvm//:llvm",
+    ] + select({
+        # The release of LLVM for Windows names zlib, zstd and libxml2 among its
+        # system libraries without shipping them. Elsewhere llvm-config
+        # names the system's.
+        "@platforms//os:windows": [
+            "@libxml2",
+            "@zlib",
+            "@zstd",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
 cc_library(
     name = "native_linker",
     srcs = ["NativeLinker.cpp"],
     hdrs = ["NativeLinker.hpp"],
     deps = [
         ":link_plan",
+        ":lld",
         "//Talac/Backend/LLVM:llvm_backend",
     ],
 )