diff --git a/.github/workflows/compiler-tier1.yml b/.github/workflows/compiler-tier1.yml index 8c01200d..8ad9992a 100644 --- a/.github/workflows/compiler-tier1.yml +++ b/.github/workflows/compiler-tier1.yml @@ -202,6 +202,44 @@ jobs: llvm_root="$(brew --prefix llvm)" echo "LLVM_ROOT=$llvm_root" >> "$GITHUB_ENV" echo "$llvm_root/bin" >> "$GITHUB_PATH" + echo "LLVM_VERSION=$("$llvm_root/bin/llvm-config" --version)" >> "$GITHUB_ENV" + + # talac links LLD's linkers statically. Homebrew's lld formula has them + # only as shared libraries over libLLVM.dylib, which would put a second + # LLVM into talac, so their static libraries are built here from the + # sources of the same LLVM version, against Homebrew's LLVM, as that + # formula builds them; the result is kept per version. + - name: Restore LLD's static libraries + id: lld-cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ${{ runner.temp }}/lld-static + key: lld-static-${{ runner.os }}-${{ runner.arch }}-${{ env.LLVM_VERSION }} + + - name: Build LLD's static libraries + if: steps.lld-cache.outputs.cache-hit != 'true' + shell: bash + run: | + source="llvm-project-${LLVM_VERSION}.src" + curl -fsSLo llvm-project.tar.xz \ + "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${source}.tar.xz" + tar -xJf llvm-project.tar.xz "$source/lld" "$source/libunwind/include" "$source/cmake" "$source/llvm/utils/lit" + cmake -G Ninja -S "$source/lld" -B lld-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$RUNNER_TEMP/lld-static" \ + -DLLVM_DIR="$LLVM_ROOT/lib/cmake/llvm" \ + -DBUILD_SHARED_LIBS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF + cmake --build lld-build --target lldCommon lldCOFF lldELF lldMachO + mkdir -p "$RUNNER_TEMP/lld-static/lib" "$RUNNER_TEMP/lld-static/include" + cp lld-build/lib/liblldCommon.a lld-build/lib/liblldCOFF.a \ + lld-build/lib/liblldELF.a lld-build/lib/liblldMachO.a "$RUNNER_TEMP/lld-static/lib/" + cp -R "$source/lld/include/lld" "$RUNNER_TEMP/lld-static/include/" + cp -R lld-build/include/lld/. "$RUNNER_TEMP/lld-static/include/lld/" 2>/dev/null || true + + - name: Use LLD's static libraries + shell: bash + run: echo "LLD_ROOT=$RUNNER_TEMP/lld-static" >> "$GITHUB_ENV" - name: Install GHC and Cabal uses: haskell/ghcup-setup@269e121e9e7528761aca5a606a53fe9d04cd6658 # v1.2.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index e56f495a..a1aef015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,22 @@ SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 ### Linking -- An executable is linked for its target rather than for the host: Windows - executables with `lld-link` from any host, for the machine of the target; - another Linux, macOS or BSD target with `clang --target= - -fuse-ld=lld`; the host's own target with `cc` as before. The runtime of - another target is looked up in `runtimes//` beside talac. +- LLD's COFF, ELF and Mach-O linkers are linked into talac, which links + executables itself. A Windows executable needs nothing from the host, and + its import libraries are written by LLVM. For Linux, macOS and the BSDs + talac takes the command line of the linker from `clang -###`, which knows + where the target's C library is, and runs it with LLD; without `clang`, a + target of the host is linked by `cc` as before. Building talac needs LLD's + static libraries and headers: the LLVM release for Windows has them, the + Linux toolchain carries them from revision 2, and on macOS they are built + from the LLVM sources against Homebrew's LLVM, because Homebrew's `lld` has + only shared libraries; `LLD_ROOT` names them where they are not beside + LLVM. + +- An executable is linked for its target rather than for the host: a Windows + executable for the machine of the target from any host, and another Linux, + macOS or BSD target through `clang --target=`. The runtime of another + target is looked up in `runtimes//` beside talac. - `--sysroot=` names the system root the linker driver links against on Linux, macOS and the BSDs; it must be a directory. A Windows target diff --git a/Documents/CLI.md b/Documents/CLI.md index 2b924197..a924abbc 100644 --- a/Documents/CLI.md +++ b/Documents/CLI.md @@ -83,13 +83,19 @@ Several sources are one program, and a program is lowered from its entry, so a c ### Linking for another target -The target, not the host, decides how an executable is linked: +Talac links executables itself: LLD's COFF, ELF and Mach-O linkers are linked into it. The target, not the host, decides +how: | Target | Linker | TalaRT, shared | TalaRT, archive | | --- | --- | --- | --- | -| Windows (`x86_64`, `aarch64`) | `lld-link`, on every host | `talart.dll` | `talart.lib` | -| the host's own system, architecture and C library | `cc`, the C compiler of the host | `libtalart.so`, `libtalart.dylib` | `libtalart.a` | -| another Linux, macOS or BSD target | `clang --target= -fuse-ld=lld` | `libtalart.so`, `libtalart.dylib` | `libtalart.a` | +| Windows (`x86_64`, `aarch64`) | LLD's `lld-link`, on every host, with nothing from the host | `talart.dll` | `talart.lib` | +| the host's own system, architecture and C library | LLD, with the command line `clang` gives for its linker | `libtalart.so`, `libtalart.dylib` | `libtalart.a` | +| another Linux, macOS or BSD target | LLD, with the command line `clang --target= -fuse-ld=lld` gives | `libtalart.so`, `libtalart.dylib` | `libtalart.a` | + +A Linux, macOS or BSD executable needs the start-up objects and the C library of its target, whose places a C compiler +knows. Talac asks `clang` for the command line of its linker (`-###`), which runs nothing, and runs that command line with +LLD. When no `clang` is on `PATH`, a target of the host is linked by `cc` and its own linker instead. Import libraries for +Windows are written by talac too. TalaRT of the host stands beside talac. TalaRT of another target stands in `runtimes//` beside it, under the normalized triple, for example `runtimes/aarch64-unknown-linux-gnu/libtalart.so`; a program that calls nothing of TalaRT @@ -101,7 +107,7 @@ talac Program.tala --entry=Demo.Program --target=x86_64-pc-windows-msvc talac Program.tala --entry=Demo.Program --target=aarch64-unknown-linux-gnu --sysroot=/opt/sysroots/arm64 ``` -`lld-link`, `cc` and `clang` are found on `PATH`. +`clang`, and `cc` where no `clang` is found, are looked up on `PATH`; no linker is. ### TalaRT diff --git a/MODULE.bazel b/MODULE.bazel index baf4edae..d40fa1ca 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,6 +10,12 @@ bazel_dep(name = "catch3", version = "0.2.0") bazel_dep(name = "google_benchmark", version = "1.9.5") bazel_dep(name = "indicators", version = "2.3.0") bazel_dep(name = "tabulate", version = "1.5.0") +# LLVM's release for Windows was built with zlib and zstd but does not ship +# them, nor libxml2; LLD, which talac links in, compresses with zlib and +# zstd and merges Windows manifests with libxml2. +bazel_dep(name = "libxml2", version = "2.15.3") +bazel_dep(name = "zlib", version = "1.3.2.bcr.3") +bazel_dep(name = "zstd", version = "1.5.7.bcr.3") # Neither header-only CLI library is currently published in the Bazel Central # Registry. Their upstream repositories are pinned as submodules beneath a diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 90b094b3..bf48b19c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -69,8 +69,12 @@ "https://bcr.bazel.build/modules/libpfm/4.11.0.bcr.1/MODULE.bazel": "e5362dadc90aab6724c83a2cc1e67cbed9c89a05d97fb1f90053c8deb1e445c8", "https://bcr.bazel.build/modules/libpfm/4.11.0.bcr.1/source.json": "0646414d9037f8aad148781dd760bec90b0b25ac12fda5e03f8aadbd6b9c61e6", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", + "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/MODULE.bazel": "a1c8bb07b5b91d971727c635f449d05623ac9608f6fe4f5f04254ea12f08e349", "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/source.json": "93f82a5ae985eb935c539bfee95e04767187818189241ac956f3ccadbdb8fb02", + "https://bcr.bazel.build/modules/package_metadata/0.0.13/MODULE.bazel": "a9cc7e81640c1e7eaca5a465cdf4b63c88d416ca998fcded3680181b1602eee2", + "https://bcr.bazel.build/modules/package_metadata/0.0.13/source.json": "f488e4534468eac223f6417dfdf8d3b7299166d36536408d37c2493ca1e8539d", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -165,14 +169,17 @@ "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://bcr.bazel.build/modules/zlib/1.3.2.bcr.3/MODULE.bazel": "0b6a197b824f4882f97f6717d81e09b10c7562023370bf7e9da6380238e9c212", + "https://bcr.bazel.build/modules/zlib/1.3.2.bcr.3/source.json": "00797abf23e06df10af2d7b78cff7d3360349e72c434329faeb532f9c358449b", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.3/MODULE.bazel": "031c2f61e3f30853704948e722b6c3c7f7989780ee37bc431747cf3dc6cbe0b3", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.3/source.json": "92e3cd6613957216e9954b59e7c944766cdd7e55e5684862a1434bd5132663de" }, "selectedYankedVersions": {}, "moduleExtensions": { "//Talac/Build/Bazel:llvm.bzl%llvm": { "general": { - "bzlTransitiveDigest": "OyOmiSyhZkmHoCehm/843yYfzXhOUcGQ001cytoFdKk=", + "bzlTransitiveDigest": "pXjfxANmpVqVW75gxVWeJf+0IhcHlXqxxjnrzDnojzI=", "usagesDigest": "O/jeFWGvuHqLFMYOMQcKWaWbSTSNRerFOnpKxqeK+Hw=", "recordedInputs": [], "generatedRepoSpecs": { diff --git a/Talac/Build/Bazel/llvm.bzl b/Talac/Build/Bazel/llvm.bzl index 211b1a70..f381c768 100644 --- a/Talac/Build/Bazel/llvm.bzl +++ b/Talac/Build/Bazel/llvm.bzl @@ -1,4 +1,28 @@ -"""Discovers an installed LLVM development tree for the C++20 backend.""" +"""Discovers an installed LLVM development tree for the C++20 backend and LLD.""" + +# The components of LLVM that LLD's COFF, ELF and Mach-O linkers use, beside +# those of the backend. They are linked into talac with LLD. +_LLD_COMPONENTS = [ + # LLD initializes every target LLVM was built for, for link-time + # optimization. + "all-targets", + "debuginfocodeview", + "debuginfodwarf", + "debuginfomsf", + "debuginfopdb", + "demangle", + "libdriver", + "lto", + "objcarcopts", + "option", + "textapi", + "windowsdriver", + "windowsmanifest", +] + +# LLD's libraries, in the order a static link needs them: the linkers before +# what they share. +_LLD_LIBRARIES = ["lldCOFF", "lldELF", "lldMachO", "lldCommon"] _COMPONENTS = [ "core", @@ -101,8 +125,15 @@ def _llvm_repository_impl(repository_ctx): library_dir, )) + # Distributed ThinLTO is a component of its own since LLVM 21; LLD uses + # it where LLVM has it. + available = repository_ctx.execute([config, "--components"], quiet = True) + lld_components = list(_LLD_COMPONENTS) + if available.return_code == 0 and "dtlto" in available.stdout.split(" "): + lld_components.append("dtlto") + libraries = repository_ctx.execute( - [config, "--link-static", "--libnames"] + _COMPONENTS, + [config, "--link-static", "--libnames"] + _COMPONENTS + lld_components, quiet = True, ) if libraries.return_code != 0: @@ -132,7 +163,7 @@ def _llvm_repository_impl(repository_ctx): ] else: system_libraries = repository_ctx.execute( - [config, "--link-static", "--system-libs"] + _COMPONENTS, + [config, "--link-static", "--system-libs"] + _COMPONENTS + lld_components, quiet = True, ) if system_libraries.return_code != 0: @@ -162,6 +193,8 @@ def _llvm_repository_impl(repository_ctx): repository_ctx.symlink(include_dir, "include") repository_ctx.symlink(library_dir, "lib") + lld_imports = _lld_imports(repository_ctx, include_dir, library_dir, library_suffix, is_windows) + support_archive = "lib/LLVMSupport.lib" if is_windows and repository_ctx.os.environ.get("TALA_LLVM_SYSTEM_ALLOCATOR") == "1": support_archive = _sanitizer_support_archive(repository_ctx, config, library_dir, library_names) @@ -194,12 +227,76 @@ cc_library( deps = {}, linkopts = {}, ) -""".format("\n".join(imports), repr(dependencies), repr(system_linkopts)) + +{} +""".format("\n".join(imports), repr(dependencies), repr(system_linkopts), lld_imports) repository_ctx.file("BUILD.bazel", build) +def _lld_imports(repository_ctx, include_dir, library_dir, library_suffix, is_windows): + """The BUILD text of the `lld` target: LLD's linkers and their headers. + + The Windows release of LLVM and the libc++ toolchain of the Linux hosts + keep LLD beside LLVM. Homebrew keeps it in a formula of its own, which + LLD_ROOT names or `brew --prefix lld` finds. + """ + prefix = "" if is_windows else "lib" + + def has_lld(include, library): + return include.get_child("lld").get_child("Common").get_child("Driver.h").exists and all([ + library.get_child("{}{}{}".format(prefix, name, library_suffix)).exists + for name in _LLD_LIBRARIES + ]) + + lld_include = include_dir + lld_library = library_dir + if not has_lld(lld_include, lld_library): + root = repository_ctx.os.environ.get("LLD_ROOT") + if not root: + brew = repository_ctx.which("brew") + if brew: + found = repository_ctx.execute([brew, "--prefix", "lld"], quiet = True) + if found.return_code == 0: + root = found.stdout.strip() + if root: + lld_include = repository_ctx.path(root).get_child("include") + lld_library = repository_ctx.path(root).get_child("lib") + if not root or not has_lld(lld_include, lld_library): + fail("LLD's static libraries and headers were not found beside LLVM. Set LLD_ROOT to a tree " + + "with lib/ and include/lld built for this LLVM; Homebrew's lld formula has only shared libraries.") + repository_ctx.symlink(lld_include, "lld-include") + repository_ctx.symlink(lld_library, "lld-lib") + header_root = "lld-include" + library_root = "lld-lib" + else: + header_root = "include" + library_root = "lib" + + text = ["""cc_library( + name = \"lld_headers\", + hdrs = glob([\"{root}/lld/**/*.h\"]), + includes = [\"{root}\"], +)""".format(root = header_root)] + previous = ":llvm" + for name in reversed(_LLD_LIBRARIES): + text.append("""cc_import( + name = {target}, + static_library = {library}, + deps = [{previous}, \":lld_headers\"], +)""".format( + target = _quote(name), + library = _quote("{}/{}{}{}".format(library_root, prefix, name, library_suffix)), + previous = _quote(previous), + )) + previous = ":" + name + text.append("""cc_library( + name = \"lld\", + deps = [{}], +)""".format(", ".join([_quote(":" + name) for name in _LLD_LIBRARIES[:-1]]))) + return "\n\n".join(text) + _llvm_repository = repository_rule( implementation = _llvm_repository_impl, - environ = ["LLVM_ROOT", "PATH", "TALA_LLVM_SYSTEM_ALLOCATOR"], + environ = ["LLD_ROOT", "LLVM_ROOT", "PATH", "TALA_LLVM_SYSTEM_ALLOCATOR"], local = True, ) diff --git a/Talac/Linker/BUILD.bazel b/Talac/Linker/BUILD.bazel index e5d332ed..15759c1c 100644 --- a/Talac/Linker/BUILD.bazel +++ b/Talac/Linker/BUILD.bazel @@ -14,12 +14,35 @@ cc_library( ], ) +# LLD's COFF, ELF and Mach-O linkers, linked into talac so that it links +# executables itself. +cc_library( + name = "lld", + srcs = ["Lld.cpp"], + hdrs = ["Lld.hpp"], + deps = [ + "@llvm//:lld", + "@llvm//:llvm", + ] + select({ + # The release of LLVM for Windows names zlib, zstd and libxml2 among its + # system libraries without shipping them. Elsewhere llvm-config + # names the system's. + "@platforms//os:windows": [ + "@libxml2", + "@zlib", + "@zstd", + ], + "//conditions:default": [], + }), +) + cc_library( name = "native_linker", srcs = ["NativeLinker.cpp"], hdrs = ["NativeLinker.hpp"], deps = [ ":link_plan", + ":lld", "//Talac/Backend/LLVM:llvm_backend", ], ) diff --git a/Talac/Linker/LinkPlan.cpp b/Talac/Linker/LinkPlan.cpp index f405615f..4fc8b77b 100644 --- a/Talac/Linker/LinkPlan.cpp +++ b/Talac/Linker/LinkPlan.cpp @@ -115,6 +115,71 @@ namespace Progmasoft::Tala::Driver / std::string(llvm::Triple::normalize(triple)); } + auto + LinkerCommandOf(const std::string_view driverOutput) + -> std::optional> + { + std::optional> found; + std::size_t start = 0U; + while (start < driverOutput.size()) + { + auto end = driverOutput.find('\n', start); + if (end == std::string_view::npos) + end = driverOutput.size(); + auto line = driverOutput.substr(start, end - start); + start = end + 1U; + // Each command is one line of words in double quotes, with `\` + // escaping a quote or a backslash inside one. + std::vector words; + std::size_t index = 0U; + bool wellFormed = true; + while (index < line.size()) + { + const auto character = line[index]; + if (character == ' ' || character == '\t' || character == '\r') + { + ++index; + continue; + } + if (character != '"') + { + wellFormed = false; + break; + } + std::string word; + ++index; + while (index < line.size() && line[index] != '"') + { + if (line[index] == '\\' && index + 1U < line.size()) + ++index; + word.push_back(line[index]); + ++index; + } + if (index >= line.size()) + { + wellFormed = false; + break; + } + ++index; + words.push_back(std::move(word)); + } + if (!wellFormed || words.empty()) + continue; + // The name of the program, on whichever host wrote the path. + std::string_view program = words.front(); + if (const auto slash = program.find_last_of("/\\"); + slash != std::string_view::npos) + program.remove_prefix(slash + 1U); + if (program.ends_with(".exe")) + program.remove_suffix(4U); + if (program == "ld" || program == "ld.lld" || program == "ld64.lld" + || program == "ld64" || program == "ld.bfd" + || program == "ld.gold") + found = std::move(words); + } + return found; + } + auto PlanLink(const LinkPlanRequest &request) -> LinkPlanResult { diff --git a/Talac/Linker/LinkPlan.hpp b/Talac/Linker/LinkPlan.hpp index c70e622f..c57db4c0 100644 --- a/Talac/Linker/LinkPlan.hpp +++ b/Talac/Linker/LinkPlan.hpp @@ -88,6 +88,14 @@ namespace Progmasoft::Tala::Driver [[nodiscard]] auto CrossRuntimeDirectory(std::string_view triple) -> std::filesystem::path; + /// The linker's command line in what a C compiler driver prints for + /// `-###`: the last command it would run whose program is a linker, + /// split into its quoted words. LLD inside talac runs it in place of + /// that program. + [[nodiscard]] auto + LinkerCommandOf(std::string_view driverOutput) + -> std::optional>; + /// Choose the linker and its arguments. Nothing is run and no file is /// touched, so every target can be planned on every host. [[nodiscard]] auto diff --git a/Talac/Linker/Lld.cpp b/Talac/Linker/Lld.cpp new file mode 100644 index 00000000..e494e5b2 --- /dev/null +++ b/Talac/Linker/Lld.cpp @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2026 Progmasoft +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +#include +#include +#include +#include +#include +#include + +#include "Talac/Linker/Lld.hpp" + +// LLD's linkers are linked into talac, so a link needs no linker on the +// host. Only the three formats talac writes are linked in. +LLD_HAS_DRIVER(coff) +LLD_HAS_DRIVER(elf) +LLD_HAS_DRIVER(macho) + +namespace Progmasoft::Tala::Driver +{ + auto + RunLld(const LldFlavor flavor, const std::vector &arguments) + -> LldOutcome + { + // LLD chooses the linker by the name of the program, as its single + // executable does. + const char *name = flavor == LldFlavor::Coff ? "lld-link" + : flavor == LldFlavor::Elf ? "ld.lld" + : "ld64.lld"; + std::vector argv; + argv.reserve(arguments.size() + 1U); + argv.push_back(name); + for (const auto &argument : arguments) + argv.push_back(argument.c_str()); + + LldOutcome outcome; + llvm::raw_string_ostream messages(outcome.messages); + const auto result + = lld::lldMain(argv, + messages, + messages, + { { lld::WinLink, &lld::coff::link }, + { lld::Gnu, &lld::elf::link }, + { lld::Darwin, &lld::macho::link } }); + messages.flush(); + outcome.exitCode = result.retCode; + // A link that failed so badly that LLD's state cannot be reset + // leaves talac unable to link again; it is one process per + // compilation, so saying so is enough. + if (!result.canRunAgain) + outcome.messages += "LLD cannot link again in this process\n"; + return outcome; + } + + auto + WriteImportLibrary(const std::filesystem::path &library, + const std::string_view dll, + const std::vector &functions, + const std::string_view machine) -> std::string + { + const auto type = machine == "arm64" + ? llvm::COFF::IMAGE_FILE_MACHINE_ARM64 + : llvm::COFF::IMAGE_FILE_MACHINE_AMD64; + std::vector exports; + exports.reserve(functions.size()); + for (const auto &function : functions) + { + llvm::object::COFFShortExport entry; + entry.Name = function; + exports.push_back(std::move(entry)); + } + const auto text = library.u8string(); + const std::string path(reinterpret_cast(text.data()), + text.size()); + if (auto error = llvm::object::writeImportLibrary( + llvm::StringRef(dll.data(), dll.size()), + path, + exports, + type, + false)) + return "could not write the import library of " + std::string(dll) + + ": " + llvm::toString(std::move(error)); + return {}; + } +} // namespace Progmasoft::Tala::Driver diff --git a/Talac/Linker/Lld.hpp b/Talac/Linker/Lld.hpp new file mode 100644 index 00000000..7a095c66 --- /dev/null +++ b/Talac/Linker/Lld.hpp @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2026 Progmasoft +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include +#include +#include +#include +#include + +namespace Progmasoft::Tala::Driver +{ + /// Which of LLD's linkers runs: each writes one object format. + enum class LldFlavor : std::uint8_t + { + Coff, ///< `lld-link`, for Windows. + Elf, ///< `ld.lld`, for Linux and the BSDs. + MachO, ///< `ld64.lld`, for macOS. + }; + + /// The exit code of a link and what LLD wrote while it ran. + struct LldOutcome final + { + int exitCode{ 1 }; + std::string messages; + }; + + /// Run one of LLD's linkers inside talac, with the arguments that the + /// linker of that name takes on its command line. + [[nodiscard]] auto + RunLld(LldFlavor flavor, const std::vector &arguments) + -> LldOutcome; + + /// Write a COFF import library for functions of a DLL, for the machine + /// `lld-link` is told (`x64` or `arm64`). Gives an empty string on + /// success and the reason otherwise. + [[nodiscard]] auto + WriteImportLibrary(const std::filesystem::path &library, + std::string_view dll, + const std::vector &functions, + std::string_view machine) -> std::string; +} // namespace Progmasoft::Tala::Driver diff --git a/Talac/Linker/NativeLinker.cpp b/Talac/Linker/NativeLinker.cpp index 23f13961..b5054b95 100644 --- a/Talac/Linker/NativeLinker.cpp +++ b/Talac/Linker/NativeLinker.cpp @@ -3,12 +3,18 @@ #include #include +#include +#include +#include +#include +#include #include #include #include #include #include "Talac/Linker/LinkPlan.hpp" +#include "Talac/Linker/Lld.hpp" #include "Talac/Linker/NativeLinker.hpp" #ifdef _WIN32 @@ -236,6 +242,19 @@ namespace Progmasoft::Tala::Driver #endif } + /// The arguments of a host link with the host's triple in front, so + /// that clang asked for the command line answers for the target + /// the objects were written for. + [[nodiscard]] auto + WithTarget(const std::string &target, + const std::vector &arguments) + -> std::vector + { + std::vector result{ "--target=" + target }; + result.insert(result.end(), arguments.begin(), arguments.end()); + return result; + } + [[nodiscard]] auto StartFailure(const std::string &program) -> std::string { @@ -323,64 +342,83 @@ namespace Progmasoft::Tala::Driver } }; - /// Writes the import library of the kernel32 functions the runtime - /// uses, for the given machine, beside the output. + /// Writes the import library a list of imports describes, for the + /// given machine, beside the output. /// /// A Windows executable is linked without the libraries of a C /// runtime or of a Windows SDK, so there is no `kernel32.lib` to - /// name. `lld-link` makes an import library from a list of names, - /// which is all that is needed to call a function of a system - /// library, on any host. + /// name, and talart.dll comes with no import library. LLVM writes + /// one from a list of names, which is all that is needed to call a + /// function of a DLL, on any host. [[nodiscard]] auto - WriteSystemImports(const std::filesystem::path &output, - const std::string_view imports, - const std::string &linker, - const std::string &machine, - Scratch &scratch, - std::string &diagnostic) + WriteImports(const std::filesystem::path &output, + const std::string_view imports, + const std::string &machine, + Scratch &scratch, + std::string &diagnostic) -> std::optional { - auto definition = output; - definition += ".imports.def"; + // The list is `LIBRARY `, `EXPORTS` and one name a line. + std::string dll; + std::vector functions; + std::size_t start = 0U; + while (start < imports.size()) + { + auto end = imports.find('\n', start); + if (end == std::string_view::npos) + end = imports.size(); + auto line = imports.substr(start, end - start); + start = end + 1U; + while (!line.empty() && line.front() == ' ') + line.remove_prefix(1U); + if (line.starts_with("LIBRARY ")) + dll = std::string(line.substr(8U)); + else if (!line.empty() && line != "EXPORTS") + functions.emplace_back(line); + } auto library = output; library += ".imports.lib"; - scratch.paths.push_back(definition); scratch.paths.push_back(library); - { - std::ofstream stream(definition, - std::ios::binary | std::ios::trunc); - stream.write(imports.data(), - static_cast(imports.size())); - if (!stream) - { - diagnostic = "could not write the list of system imports " - "for the native link"; - return std::nullopt; - } - } - const auto utf8 = [](const std::filesystem::path &path) { - const auto text = path.u8string(); - return std::string(reinterpret_cast(text.data()), - text.size()); - }; - const auto status = Run(linker, - { "/lib", - "/nologo", - "/machine:" + machine, - "/def:" + utf8(definition), - "/out:" + utf8(library) }); - if (status != 0) - { - diagnostic = status == -1 - ? StartFailure(linker) - : linker - + " could not make the import library " - "of the system functions; exit code " - + std::to_string(status); + diagnostic = WriteImportLibrary(library, dll, functions, machine); + if (!diagnostic.empty()) return std::nullopt; - } return library; } + + /// The linker's command line a C compiler driver would run for the + /// given arguments, which it prints for `-###` without running + /// anything. Nothing when the driver cannot be started or prints no + /// linker. + [[nodiscard]] auto + DriverLinkerCommand(const std::string &driver, + const std::vector &arguments) + -> std::optional> + { + auto program = llvm::sys::findProgramByName(driver); + if (!program) + return std::nullopt; + llvm::SmallString<128> output; + if (llvm::sys::fs::createTemporaryFile("talac-driver", + "txt", + output)) + return std::nullopt; + llvm::FileRemover remover(output); + std::vector argv{ *program, "-###" }; + for (const auto &argument : arguments) + argv.emplace_back(argument); + const std::optional redirects[] + = { llvm::StringRef(""), llvm::StringRef(""), output.str() }; + if (llvm::sys::ExecuteAndWait(*program, + argv, + std::nullopt, + redirects) + != 0) + return std::nullopt; + auto text = llvm::MemoryBuffer::getFile(output); + if (!text) + return std::nullopt; + return LinkerCommandOf((*text)->getBuffer()); + } } // namespace auto @@ -443,10 +481,9 @@ namespace Progmasoft::Tala::Driver && planRequest.runtimeLibrary) { std::string importDiagnostic; - const auto imports = WriteSystemImports( + const auto imports = WriteImports( request.outputPath, planRequest.sharedRuntime ? kRuntimeImports : kSystemImports, - planned.plan->program, planned.plan->machine, scratch, importDiagnostic); @@ -459,7 +496,54 @@ namespace Progmasoft::Tala::Driver } const auto &plan = *planned.plan; - const auto status = Run(plan.program, plan.arguments); + // LLD inside talac links. A Windows executable needs nothing more + // than the plan. Another target needs the start-up objects and the + // C library a driver knows the places of: clang prints the command + // line of its linker for `-###`, and LLD runs it. Only when no + // clang is there to ask is a target of the host linked by its own + // C compiler, which runs its own linker. + std::optional linked; + if (plan.linker == Linker::LldLink) + linked = RunLld(LldFlavor::Coff, plan.arguments); + else + { + const auto command = DriverLinkerCommand( + plan.linker == Linker::CrossDriver ? plan.program + : std::string("clang"), + plan.linker == Linker::CrossDriver + ? plan.arguments + : WithTarget(target, plan.arguments)); + if (command) + linked = RunLld(TargetObjectFormat(target) + == Backend::LLVM::ObjectFormat::MachO + ? LldFlavor::MachO + : LldFlavor::Elf, + std::vector(command->begin() + 1, + command->end())); + else if (plan.linker == Linker::CrossDriver) + return { -1, + "could not ask '" + plan.program + "' how to link for " + + target + + "; an executable for another target needs " + "clang to know where the target's C library " + "is" }; + } + if (linked) + { + if (linked->exitCode != 0) + return { linked->exitCode, + "LLD could not link the executable:\n" + + linked->messages + + (planRequest.runtimeLibrary + ? std::string() + : "the Tala runtime library was not found " + "at " + + runtime.string() + + ", and a program that uses " + "strings, closures, the console or " + "tests needs it") }; + } + const auto status = linked ? 0 : Run(plan.program, plan.arguments); // Waiting is deliberate: success cannot be reported, and `run` cannot // begin, until the linker has closed and finalized the image. if (status == -1) diff --git a/Talac/Linker/Tests/BUILD.bazel b/Talac/Linker/Tests/BUILD.bazel index 63546600..fb0ccdae 100644 --- a/Talac/Linker/Tests/BUILD.bazel +++ b/Talac/Linker/Tests/BUILD.bazel @@ -11,3 +11,17 @@ cc_binary( "@catch3//src/Progmasoft:catch3", ], ) + +# LLD runs inside this test as it does inside talac. It links a Windows +# executable from an object LLVM generates, so every host links one, and a +# Windows host also runs it. +cc_binary( + name = "lld_tests", + srcs = ["LldTests.cpp"], + deps = [ + "//Talac/Linker:lld", + "@catch3//:catch2_main", + "@catch3//src/Progmasoft:catch3", + "@llvm//:llvm", + ], +) diff --git a/Talac/Linker/Tests/LinkPlanTests.cpp b/Talac/Linker/Tests/LinkPlanTests.cpp index 17bbe4c0..18a271d0 100644 --- a/Talac/Linker/Tests/LinkPlanTests.cpp +++ b/Talac/Linker/Tests/LinkPlanTests.cpp @@ -186,3 +186,43 @@ TEST_CASE("a link that cannot be planned says why") CHECK_FALSE( Driver::PlanLink(Request("wasm32-unknown-unknown", kLinux)).plan); } + +TEST_CASE("the linker's command line is read from what clang prints for -###") +{ + // clang prints its version, then each command as quoted words; a + // backslash escapes a quote or a backslash inside one. + constexpr std::string_view kLinuxOutput + = "clang version 22.1.6\n" + "Target: x86_64-unknown-linux-gnu\n" + "Thread model: posix\n" + " \"/usr/bin/ld\" \"-pie\" \"-m\" \"elf_x86_64\" \"-o\" " + "\"out/Program\" " + "\"/usr/lib/x86_64-linux-gnu/Scrt1.o\" \"out/Program.o\" \"-lc\"\n"; + const auto elf = Driver::LinkerCommandOf(kLinuxOutput); + REQUIRE(elf); + CHECK(elf->front() == "/usr/bin/ld"); + CHECK(elf->size() == 9U); + CHECK(elf->back() == "-lc"); + + constexpr std::string_view kWindowsHost + = " \"C:\\\\LLVM\\\\bin\\\\clang.exe\" \"-cc1\" \"-triple\" \"x\"\n" + " \"C:\\\\LLVM\\\\bin\\\\ld.lld.exe\" " + "\"--sysroot=C:\\\\roots\\\\arm\" " + "\"-o\" \"a b\" \"q\\\"uote\"\n"; + const auto cross = Driver::LinkerCommandOf(kWindowsHost); + REQUIRE(cross); + CHECK(cross->front() == "C:\\LLVM\\bin\\ld.lld.exe"); + CHECK((*cross)[1] == "--sysroot=C:\\roots\\arm"); + CHECK((*cross)[3] == "a b"); + CHECK((*cross)[4] == "q\"uote"); + + constexpr std::string_view kMacOutput + = " \"/usr/bin/ld\" \"-demangle\" \"-dynamic\" \"-arch\" \"arm64\" " + "\"-platform_version\" \"macos\" \"26.0.0\" \"26.0\" \"-o\" \"P\"\n"; + REQUIRE(Driver::LinkerCommandOf(kMacOutput)); + + // No linker, or a line that is not a command, gives nothing. + CHECK_FALSE(Driver::LinkerCommandOf("clang version 22.1.6\n")); + CHECK_FALSE(Driver::LinkerCommandOf(" \"/usr/bin/as\" \"-o\" \"x.o\"\n")); + CHECK_FALSE(Driver::LinkerCommandOf(" \"/usr/bin/ld\" \"unterminated\n")); +} diff --git a/Talac/Linker/Tests/LldTests.cpp b/Talac/Linker/Tests/LldTests.cpp new file mode 100644 index 00000000..b2789722 --- /dev/null +++ b/Talac/Linker/Tests/LldTests.cpp @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: 2026 Progmasoft +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Talac/Linker/Lld.hpp" + +// LLD is linked into talac: these tests run it inside the test process. The +// object a Windows link reads is generated here by LLVM, so the link needs +// nothing from the host but the test itself. + +namespace +{ + namespace Driver = Progmasoft::Tala::Driver; + + /// A directory of its own for one test, removed afterwards. + struct Directory final + { + std::filesystem::path path; + + Directory() + { + llvm::SmallString<128> created; + if (!llvm::sys::fs::createUniqueDirectory("talac-lld", created)) + path = created.str().str(); + } + Directory(const Directory &) = delete; + Directory(Directory &&) = delete; + auto + operator=(const Directory &) -> Directory & = delete; + auto + operator=(Directory &&) -> Directory & = delete; + ~Directory() + { + std::error_code ignored; + std::filesystem::remove_all(path, ignored); + } + }; + + /// A COFF object for x86-64 Windows whose `mainCRTStartup` returns 42. + [[nodiscard]] auto + WriteEntryObject(const std::filesystem::path &object) -> bool + { + LLVMInitializeX86TargetInfo(); + LLVMInitializeX86Target(); + LLVMInitializeX86TargetMC(); + LLVMInitializeX86AsmPrinter(); + const llvm::Triple triple("x86_64-pc-windows-msvc"); + std::string error; + const auto *target = llvm::TargetRegistry::lookupTarget(triple, error); + if (target == nullptr) + return false; + std::unique_ptr machine( + target->createTargetMachine(triple, + "x86-64", + "", + llvm::TargetOptions{}, + std::nullopt)); + llvm::LLVMContext context; + llvm::Module module("entry", context); + module.setTargetTriple(triple); + module.setDataLayout(machine->createDataLayout()); + auto *int32 = llvm::Type::getInt32Ty(context); + auto *entry + = llvm::Function::Create(llvm::FunctionType::get(int32, false), + llvm::GlobalValue::ExternalLinkage, + "mainCRTStartup", + module); + llvm::IRBuilder<> builder( + llvm::BasicBlock::Create(context, "entry", entry)); + builder.CreateRet(llvm::ConstantInt::get(int32, 42)); + + std::error_code openError; + llvm::raw_fd_ostream stream(object.string(), openError); + if (openError) + return false; + llvm::legacy::PassManager passes; + if (machine->addPassesToEmitFile(passes, + stream, + nullptr, + llvm::CodeGenFileType::ObjectFile)) + return false; + passes.run(module); + stream.flush(); + return true; + } +} // namespace + +TEST_CASE("an import library is written for the functions of a DLL") +{ + const Directory directory; + const auto library = directory.path / "talart.dll.lib"; + for (const auto *machine : { "x64", "arm64" }) + { + CHECK(Driver::WriteImportLibrary(library, + "talart.dll", + { "tala_console_write", "memset" }, + machine) + .empty()); + std::error_code error; + CHECK(std::filesystem::file_size(library, error) > 0U); + } +} + +TEST_CASE("a link LLD refuses fails with what LLD said") +{ + const Directory directory; + const auto outcome + = Driver::RunLld(Driver::LldFlavor::Coff, + { "/nologo", + "/out:" + (directory.path / "a.exe").string(), + (directory.path / "missing.obj").string() }); + CHECK(outcome.exitCode != 0); + CHECK(outcome.messages.find("missing.obj") != std::string::npos); + // LLD can run again after a refused link. + const auto again = Driver::RunLld(Driver::LldFlavor::Elf, { "--version" }); + CHECK(again.exitCode == 0); + CHECK(again.messages.find("LLD") != std::string::npos); +} + +TEST_CASE("LLD inside the process links a Windows executable") +{ + const Directory directory; + const auto object = directory.path / "entry.obj"; + REQUIRE(WriteEntryObject(object)); + const auto executable = directory.path / "entry.exe"; + const auto outcome = Driver::RunLld(Driver::LldFlavor::Coff, + { "/nologo", + "/entry:mainCRTStartup", + "/subsystem:console", + "/nodefaultlib", + "/machine:x64", + "/out:" + executable.string(), + object.string() }); + INFO(outcome.messages); + REQUIRE(outcome.exitCode == 0); + std::error_code error; + CHECK(std::filesystem::file_size(executable, error) > 0U); +#if defined(_WIN32) && defined(_M_X64) + // The program runs, and the exit code is the one its entry returns. + CHECK(std::system(("\"" + executable.string() + "\"").c_str()) == 42); +#endif +} diff --git a/helpers/containers/fedora-43/Containerfile b/helpers/containers/fedora-43/Containerfile index 31ce52ea..8748f309 100644 --- a/helpers/containers/fedora-43/Containerfile +++ b/helpers/containers/fedora-43/Containerfile @@ -40,8 +40,8 @@ RUN set -eu; \ *) echo "no LLVM toolchain archive with libc++ for ${TARGETARCH} yet" >&2; exit 1 ;; \ esac; \ curl -fsSL -o /tmp/llvm.tar.zst \ - "https://github.com/Progmasoft/tala/releases/download/toolchain-llvm-22.1.6-libcxx-1/llvm-22.1.6-libcxx-fedora-43-x86_64.tar.zst"; \ - echo "5355714911c1d1dcf7f3486d2f2c0b3a468be838314fbe42b49a07b1e86db7b5 /tmp/llvm.tar.zst" | sha256sum -c -; \ + "https://github.com/Progmasoft/tala/releases/download/toolchain-llvm-22.1.6-libcxx-2/llvm-22.1.6-libcxx-fedora-43-x86_64.tar.zst"; \ + echo "1621b84ec14bc0212b9c366df84be4d12d80235c3843402216689e0498c6169d /tmp/llvm.tar.zst" | sha256sum -c -; \ tar --zstd -xf /tmp/llvm.tar.zst -C /opt; \ mv /opt/llvm-22.1.6-libcxx-fedora-43-x86_64 /opt/llvm-libcxx; \ rm /tmp/llvm.tar.zst; \ diff --git a/helpers/containers/ubuntu-26.04/Containerfile b/helpers/containers/ubuntu-26.04/Containerfile index b2287e49..cb6e82cb 100644 --- a/helpers/containers/ubuntu-26.04/Containerfile +++ b/helpers/containers/ubuntu-26.04/Containerfile @@ -41,8 +41,8 @@ RUN set -eu; \ *) echo "no LLVM toolchain archive with libc++ for ${TARGETARCH} yet" >&2; exit 1 ;; \ esac; \ curl -fsSL -o /tmp/llvm.tar.zst \ - "https://github.com/Progmasoft/tala/releases/download/toolchain-llvm-22.1.6-libcxx-1/llvm-22.1.6-libcxx-ubuntu-26.04-x86_64.tar.zst"; \ - echo "5e8a4c1f52b7d7eedf155b76bd057cec079e11daa66822d1f6af1109c551694b /tmp/llvm.tar.zst" | sha256sum -c -; \ + "https://github.com/Progmasoft/tala/releases/download/toolchain-llvm-22.1.6-libcxx-2/llvm-22.1.6-libcxx-ubuntu-26.04-x86_64.tar.zst"; \ + echo "ed9bbc64f7230a2f5955b8b433559711ff3fe34499d8aba453425b56c69d4604 /tmp/llvm.tar.zst" | sha256sum -c -; \ tar --zstd -xf /tmp/llvm.tar.zst -C /opt; \ mv /opt/llvm-22.1.6-libcxx-ubuntu-26.04-x86_64 /opt/llvm-libcxx; \ rm /tmp/llvm.tar.zst; \ diff --git a/helpers/internal/development/build.go b/helpers/internal/development/build.go index d9505181..b134ea94 100644 --- a/helpers/internal/development/build.go +++ b/helpers/internal/development/build.go @@ -25,6 +25,7 @@ var nativeTargets = []string{ "//Talac/Core/Tests:core_pipeline_tests", "//Talac/Diagnostic/Tests:diagnostic_protocol_tests", "//Talac/Linker/Tests:link_plan_tests", + "//Talac/Linker/Tests:lld_tests", "//Talac/Driver/Tests:artifact_wire_tests", "//Talac/Driver/Tests:closure_pipeline_tests", "//Talac/Driver/Tests:project_artifact_tests", diff --git a/helpers/internal/development/llvm_toolchain.go b/helpers/internal/development/llvm_toolchain.go index bbfef276..31fbeea6 100644 --- a/helpers/internal/development/llvm_toolchain.go +++ b/helpers/internal/development/llvm_toolchain.go @@ -25,7 +25,7 @@ import ( // against the digest recorded here before it is used. const ( llvmToolchainVersion = "22.1.6" - llvmToolchainRevision = 1 + llvmToolchainRevision = 2 ) // llvmToolchainArchitecture is the architecture the archives are chosen @@ -34,8 +34,8 @@ var llvmToolchainArchitecture = runtime.GOARCH // llvmToolchainDigests holds the SHA-256 of the archive of each Linux host. var llvmToolchainDigests = map[string]string{ - "ubuntu-26.04": "5e8a4c1f52b7d7eedf155b76bd057cec079e11daa66822d1f6af1109c551694b", - "fedora-43": "5355714911c1d1dcf7f3486d2f2c0b3a468be838314fbe42b49a07b1e86db7b5", + "ubuntu-26.04": "ed9bbc64f7230a2f5955b8b433559711ff3fe34499d8aba453425b56c69d4604", + "fedora-43": "1621b84ec14bc0212b9c366df84be4d12d80235c3843402216689e0498c6169d", } // llvmToolchainPlatform names the archive that suits a Linux host. diff --git a/helpers/internal/development/llvm_toolchain_test.go b/helpers/internal/development/llvm_toolchain_test.go index 9c41038a..e46f7a9f 100644 --- a/helpers/internal/development/llvm_toolchain_test.go +++ b/helpers/internal/development/llvm_toolchain_test.go @@ -46,7 +46,7 @@ func TestTheLinuxHostsHaveAnArchiveAndADigest(t *testing.T) { } func TestTheArchiveComesFromTheImmutableToolchainRelease(t *testing.T) { - want := "https://github.com/Progmasoft/tala/releases/download/toolchain-llvm-22.1.6-libcxx-1/llvm-22.1.6-libcxx-fedora-43-x86_64.tar.zst" + want := "https://github.com/Progmasoft/tala/releases/download/toolchain-llvm-22.1.6-libcxx-2/llvm-22.1.6-libcxx-fedora-43-x86_64.tar.zst" if got := llvmToolchainURL("fedora-43"); got != want { t.Fatalf("URL = %q, want %q", got, want) }