diff --git a/Gemfile.lock b/Gemfile.lock index 35ea07f..2973f04 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - prebake (0.3.2) + prebake (0.3.3) GEM remote: https://rubygems.org/ @@ -91,7 +91,7 @@ CHECKSUMS mocha (3.1.0) sha256=75f42d69ebfb1f10b32489dff8f8431d37a418120ecdfc07afe3bc183d4e1d56 parallel (1.28.0) sha256=33e6de1484baf2524792d178b0913fc8eb94c628d6cfe45599ad4458c638c970 parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54 - prebake (0.3.2) + prebake (0.3.3) prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623 racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f diff --git a/README.md b/README.md index 50138ac..6ef7a34 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ All configuration is done through environment variables. No code changes require 2. Prebake intercepts `Gem::Ext::Builder#build_extensions` before compilation starts. 3. A cache key is generated from the gem name, version, platform (for example `aarch64-linux`), and Ruby ABI version (e.g., `4.0`). 4. The plugin checks the configured backend for a precompiled binary matching that key. -5. **Cache hit**: the prebuilt `.so`/`.bundle` files are extracted directly, with no compiler needed. +5. **Cache hit**: the prebuilt `.so`/`.bundle` files are extracted to the same places a source build puts them, with no compiler needed. 6. **Cache miss**: Bundler compiles from source as usual (no impact, same as without the plugin). ### Publishing compiled gems (publisher) @@ -227,7 +227,7 @@ PREBAKE_LOG_LEVEL=debug bundle install # full diagnostics - `Backend initialization failed`: check your backend URL and credentials. The plugin disables itself gracefully and Bundler continues normally. - `Checksum mismatch`: the downloaded binary doesn't match the stored SHA-256. The plugin automatically falls back to compiling from source. - `LoadError: cannot load such file -- bigdecimal.so` (Ruby 4.0): RubyGems 4 [stopped copying `.so` files into `lib/`](https://github.com/ruby/rubygems/pull/9240), and Bundler 2.5.x doesn't properly resolve the new extension directory layout. This is a Bundler bug, not a prebake issue. Fix it by upgrading Bundler: `gem install bundler && bundle update --bundler`. If you're using `ruby/setup-ruby` with `bundler-cache: true` in CI, also delete the stale Actions cache so the extensions are rebuilt with the new Bundler. -- `LoadError: Could not open library '.../sassc-2.4.0/lib/sassc/libsass.so'` (Ruby 4.0): same root cause as above — RubyGems 4 [stopped copying `.so` files into `lib/`](https://github.com/ruby/rubygems/pull/9240) — but `sassc` resolves `libsass.so` via FFI's absolute-path `File.expand_path("libsass.#{dl_ext}", __dir__)`, which dlopens the literal path under `gem_dir/lib/sassc/`. Bundler can't make `$LOAD_PATH` rescue this, so a Bundler upgrade alone won't fix it. Reproducible without prebake. Switch to [`sassc-embedded`](https://rubygems.org/gems/sassc-embedded), a drop-in replacement that ships a precompiled `libsass` and avoids the FFI absolute-path dance, or pin Ruby 3.x. +- `LoadError: Could not open library '.../sassc-2.4.0/lib/sassc/libsass.so'`, or `cannot find 'llhttp-ext' library` with `ffi-compiler` below 1.4: prebake before 0.3.3 put cached binaries only in the gem's extension directory, but `sassc` and `llhttp-ffi` load their library over FFI from a fixed path inside the gem directory. Upgrade prebake, then reinstall the affected gem once with `bundle pristine sassc` (or `llhttp-ffi`). Bundler treats the broken install as complete, so a plain `bundle install` leaves it in place. - To disable for a single run: `PREBAKE_ENABLED=false bundle install`. ## Compatibility diff --git a/lib/prebake/extractor.rb b/lib/prebake/extractor.rb index f72b4ed..4e553f2 100644 --- a/lib/prebake/extractor.rb +++ b/lib/prebake/extractor.rb @@ -22,6 +22,7 @@ def self.install(gem_path, spec) dest = File.join(spec.extension_dir, normalized_path(binary, tmpdir)) FileUtils.mkdir_p(File.dirname(dest)) FileUtils.cp(binary, dest) + restore_in_gem_dir(binary, tmpdir, spec) extracted_count += 1 end end @@ -60,5 +61,19 @@ def self.normalized_path(binary, tmpdir) relative.sub(%r{\Aextensions?/[^/]+/[^/]+/}, "") end + + # A source build leaves binaries in the gem dir too: ffi-compiler builds in + # place under ext/, and RubyGems copies extconf output into lib/. FFI gems + # dlopen those exact paths (llhttp-ffi, sassc). Only ext/ and lib/ are + # restored, because self-hosted cached gems put extension_dir binaries at + # the gem root, where no build would leave them. + def self.restore_in_gem_dir(binary, tmpdir, spec) + relative = binary.delete_prefix("#{tmpdir}/") + return unless relative.start_with?("ext/", "lib/") + + dest = File.join(spec.full_gem_path, relative) + FileUtils.mkdir_p(File.dirname(dest)) + FileUtils.cp(binary, dest) + end end end diff --git a/prebake.gemspec b/prebake.gemspec index 8610628..e5155ce 100644 --- a/prebake.gemspec +++ b/prebake.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prebake" - spec.version = "0.3.2" + spec.version = "0.3.3" spec.authors = ["Thejus Paul"] spec.email = ["thejuspaul@pm.me"] diff --git a/test/extractor_test.rb b/test/extractor_test.rb index b2d1c10..33f1fdf 100644 --- a/test/extractor_test.rb +++ b/test/extractor_test.rb @@ -9,7 +9,9 @@ class ExtractorTest < Minitest::Test def setup @tmpdir = Dir.mktmpdir @extension_dir = File.join(@tmpdir, "extensions") + @gem_dir = File.join(@tmpdir, "gem") FileUtils.mkdir_p(@extension_dir) + FileUtils.mkdir_p(@gem_dir) end def teardown @@ -73,6 +75,7 @@ def test_strips_ext_prefix_from_legacy_cached_gems spec = mock("spec") spec.stubs(:extension_dir).returns(@extension_dir) + spec.stubs(:full_gem_path).returns(@gem_dir) Prebake::Extractor.install(gem_path, spec) @@ -137,6 +140,7 @@ def test_strips_lib_prefix_from_legacy_cached_gems spec = mock("spec") spec.stubs(:extension_dir).returns(@extension_dir) + spec.stubs(:full_gem_path).returns(@gem_dir) Prebake::Extractor.install(gem_path, spec) @@ -145,6 +149,51 @@ def test_strips_lib_prefix_from_legacy_cached_gems assert_equal "legacy-binary", File.read(extracted) end + # ffi-compiler builds in place under ext/ and RubyGems copies extconf output + # into lib/. FFI gems such as llhttp-ffi and sassc dlopen those paths. + def test_restores_gem_dir_binaries_to_their_packaged_paths + gem_path = build_fake_platform_gem( + "testgem", "1.0.0", + files: { + "ext/x86_64-linux/libtestgem-ext.so" => "in-place-build-output", + "lib/testgem/libtestgem.so" => "lib-copy" + } + ) + + spec = mock("spec") + spec.stubs(:extension_dir).returns(@extension_dir) + spec.stubs(:full_gem_path).returns(@gem_dir) + + Prebake::Extractor.install(gem_path, spec) + + in_place = File.join(@gem_dir, "ext/x86_64-linux/libtestgem-ext.so") + lib_copy = File.join(@gem_dir, "lib/testgem/libtestgem.so") + assert File.exist?(in_place), "Expected #{in_place} to exist" + assert File.exist?(lib_copy), "Expected #{lib_copy} to exist" + assert_equal "in-place-build-output", File.read(in_place) + assert_equal "lib-copy", File.read(lib_copy) + end + + # Self-hosted cached gems carry extension_dir binaries at the gem root, and + # extension/ is the hosted copy of extension_dir. Neither came from gem_dir. + def test_does_not_write_extension_dir_artifacts_into_gem_dir + gem_path = build_fake_platform_gem( + "testgem", "1.0.0", + files: { + "testgem.so" => "self-hosted-root-level", + "extension/x86_64-linux/4.0.0/testgem/testgem.so" => "extension-dir-copy" + } + ) + + spec = mock("spec") + spec.stubs(:extension_dir).returns(@extension_dir) + spec.stubs(:full_gem_path).returns(@gem_dir) + + Prebake::Extractor.install(gem_path, spec) + + assert_empty Dir.children(@gem_dir) + end + def test_writes_prebake_marker_on_successful_extraction gem_path = build_fake_platform_gem( "testgem", "1.0.0",