diff --git a/Gemfile.lock b/Gemfile.lock
index 2973f04..54a1996 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    prebake (0.3.3)
+    prebake (0.3.4)
 
 GEM
   remote: https://rubygems.org/
@@ -91,7 +91,7 @@ CHECKSUMS
   mocha (3.1.0) sha256=75f42d69ebfb1f10b32489dff8f8431d37a418120ecdfc07afe3bc183d4e1d56
   parallel (1.28.0) sha256=33e6de1484baf2524792d178b0913fc8eb94c628d6cfe45599ad4458c638c970
   parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54
-  prebake (0.3.3)
+  prebake (0.3.4)
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
   public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623
   racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f
diff --git a/README.md b/README.md
index 6ef7a34..1c978ec 100644
--- a/README.md
+++ b/README.md
@@ -91,6 +91,8 @@ All configuration is done through environment variables. No code changes require
 5. **Cache hit**: the prebuilt `.so`/`.bundle` files are extracted to the same places a source build puts them, with no compiler needed.
 6. **Cache miss**: Bundler compiles from source as usual (no impact, same as without the plugin).
 
+A gem with build options, set through `bundle config set build.<gem>` or `gem install <gem> -- <options>`, is always compiled from source. Cached binaries are built with default options, so prebake skips the cache for that gem.
+
 ### Publishing compiled gems (publisher)
 
 When `PREBAKE_PUSH_ENABLED=true` (for self-hosted setups):
diff --git a/lib/prebake/ext_builder_patch.rb b/lib/prebake/ext_builder_patch.rb
index ebeb7f5..a4f0e72 100644
--- a/lib/prebake/ext_builder_patch.rb
+++ b/lib/prebake/ext_builder_patch.rb
@@ -23,6 +23,7 @@ def build_extensions
       return super unless @spec.extensions.any?
       return super unless Prebake.enabled?
       return super unless Prebake.backend # nil if config failed
+      return super unless @build_args.empty? # cached binaries are compiled with default options
 
       if Prebake.optional_native_extension?(@spec.name) && !Prebake.libruby_available?
         Logger.warn "#{@spec.name}: native extension skipped (optional gem, libruby.so absent on static Ruby build)"
diff --git a/prebake.gemspec b/prebake.gemspec
index e5155ce..7919f1a 100644
--- a/prebake.gemspec
+++ b/prebake.gemspec
@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "prebake"
-  spec.version       = "0.3.3"
+  spec.version       = "0.3.4"
   spec.authors       = ["Thejus Paul"]
   spec.email         = ["thejuspaul@pm.me"]
 
diff --git a/test/ext_builder_patch_test.rb b/test/ext_builder_patch_test.rb
index 550fc8f..f364064 100644
--- a/test/ext_builder_patch_test.rb
+++ b/test/ext_builder_patch_test.rb
@@ -103,6 +103,20 @@ def test_skips_gems_without_extensions
     build(spec)
   end
 
+  def test_builds_from_source_when_build_args_are_set
+    spec = make_spec
+    gem_path = fake_gem_path
+    backend = mock("backend")
+    backend.stubs(:fetch_checksum).with(cache_key).returns(Digest::SHA256.file(gem_path).hexdigest)
+    backend.stubs(:fetch).with(cache_key).returns(gem_path)
+    Prebake.backend = backend
+    Prebake::Extractor.expects(:install).never
+    builder = Gem::Ext::Builder.new(spec, ["--with-pg-config=/opt/pg/bin/pg_config"])
+
+    assert_raises(Gem::Ext::BuildError) { builder.build_extensions }
+    refute File.exist?(@build_complete), "gem_build_complete marker should not be written"
+  end
+
   def test_deletes_cache_and_falls_back_when_no_binaries_in_cached_gem
     spec, backend, gem_path = stub_verified_cache_hit
     backend.expects(:delete).with(cache_key)