diff --git a/Gemfile.lock b/Gemfile.lock
index 54a1996..b75811e 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    prebake (0.3.4)
+    prebake (0.3.5)
 
 GEM
   remote: https://rubygems.org/
@@ -91,7 +91,7 @@ CHECKSUMS
   mocha (3.1.0) sha256=75f42d69ebfb1f10b32489dff8f8431d37a418120ecdfc07afe3bc183d4e1d56
   parallel (1.28.0) sha256=33e6de1484baf2524792d178b0913fc8eb94c628d6cfe45599ad4458c638c970
   parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54
-  prebake (0.3.4)
+  prebake (0.3.5)
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
   public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623
   racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f
diff --git a/README.md b/README.md
index 1c978ec..8423a11 100644
--- a/README.md
+++ b/README.md
@@ -102,6 +102,8 @@ When `PREBAKE_PUSH_ENABLED=true` (for self-hosted setups):
 3. The gem and checksum are uploaded to the backend in the background.
 4. Future `bundle install` runs by anyone on the same platform get the precompiled version.
 
+A gem compiled with build options is never published. Everyone on the same platform would receive it as the default build of that gem, so only gems compiled with default options are uploaded.
+
 ## Supported platforms (cloud service)
 
 | Platform | Architecture | OS |
diff --git a/lib/prebake/async_publisher.rb b/lib/prebake/async_publisher.rb
index 902b683..ee8d821 100644
--- a/lib/prebake/async_publisher.rb
+++ b/lib/prebake/async_publisher.rb
@@ -54,6 +54,8 @@ def self.wait_for_completion(timeout: 120)
     end
 
     def self.build_gem(spec, backend)
+      return nil unless spec.build_args.empty? # consumers expect a cache entry to be a default build
+
       platform = Platform.generalized
       cache_key = CacheKey.for(spec.name, spec.version.to_s, platform)
 
diff --git a/prebake.gemspec b/prebake.gemspec
index 7919f1a..89f3153 100644
--- a/prebake.gemspec
+++ b/prebake.gemspec
@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "prebake"
-  spec.version       = "0.3.4"
+  spec.version       = "0.3.5"
   spec.authors       = ["Thejus Paul"]
   spec.email         = ["thejuspaul@pm.me"]
 
diff --git a/test/async_publisher_test.rb b/test/async_publisher_test.rb
index 1b7a330..3cb7b4a 100644
--- a/test/async_publisher_test.rb
+++ b/test/async_publisher_test.rb
@@ -15,8 +15,8 @@ def teardown
     FileUtils.rm_rf(@tmpdir)
   end
 
-  def mock_spec(name, version)
-    stub(name:, version: Gem::Version.new(version))
+  def mock_spec(name, version, build_args: [])
+    stub(name:, version: Gem::Version.new(version), build_args:)
   end
 
   def temp_gem_path(name = "fake")
@@ -101,6 +101,18 @@ def test_pushes_without_check_when_max_glibc_unset
     Prebake::AsyncPublisher.wait_for_completion(timeout: 5)
   end
 
+  def test_skips_push_when_gem_was_compiled_with_build_options
+    spec = mock_spec("pg", "1.5.4", build_args: ["--with-pg-config=/opt/pg/bin/pg_config"])
+    backend = stub(exists?: false)
+    backend.expects(:push).never
+
+    builder = stub(build: temp_gem_path, checksum: "sha")
+    Prebake::PlatformGemBuilder.stubs(:new).with(spec).returns(builder)
+
+    Prebake::AsyncPublisher.enqueue(spec, backend)
+    Prebake::AsyncPublisher.wait_for_completion(timeout: 5)
+  end
+
   def test_skips_push_when_built_gem_needs_libruby_on_static_ruby
     spec = mock_spec("bootsnap", "1.23.0")
     gem_path = temp_gem_path