From a394aeadfe451f3d574d716cdb2e3926a6b23d05 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 00:27:08 +0100 Subject: [PATCH 01/20] added concurrent, cumulative and windowed stategies for redis_mutex --- lib/redis_mutex.rb | 112 ++++++++++++++++++---------- lib/redis_mutex/concurrent_mutex.rb | 34 +++++++++ lib/redis_mutex/cumulative_mutex.rb | 75 +++++++++++++++++++ lib/redis_mutex/standard_mutex.rb | 66 ++++++++++++++++ lib/redis_mutex/windowed_mutex.rb | 57 ++++++++++++++ 5 files changed, 306 insertions(+), 38 deletions(-) create mode 100644 lib/redis_mutex/concurrent_mutex.rb create mode 100644 lib/redis_mutex/cumulative_mutex.rb create mode 100644 lib/redis_mutex/standard_mutex.rb create mode 100644 lib/redis_mutex/windowed_mutex.rb diff --git a/lib/redis_mutex.rb b/lib/redis_mutex.rb index cd8039d..03c80a1 100644 --- a/lib/redis_mutex.rb +++ b/lib/redis_mutex.rb @@ -1,3 +1,5 @@ +require 'securerandom' + class RedisMutex < RedisClassy # # Options @@ -8,8 +10,25 @@ class RedisMutex < RedisClassy # It is recommended that you do NOT go below 0.01. (default: 0.1) # :expire => Specify in seconds when the lock should forcibly be removed when something went wrong # with the one who held the lock. (default: 10) + # :limit => Specify how many times the provided block can be executed. (default: 1) + # When type is :cumulative or :windowed it means executions over the expire period + # When type is :concurrent it means concurrent executions + # :type => Specify the type of the mutex. (default: :concurrent) [:cumulative, :windowed, :concurrent] + # :concurrent + limit = 1 is the same as the original RedisMutex + # :concurrent is to limit parallel or concurrent executions of the block + # :cumulative is to limit total executions of the block over the past expire seconds + # :windowed is to limit executions of the block in a window that represents expire full seconds # autoload :Macro, 'redis_mutex/macro' + autoload :StandardMutex, 'redis_mutex/standard_mutex' + autoload :CumulativeMutex, 'redis_mutex/cumulative_mutex' + autoload :ConcurrentMutex, 'redis_mutex/concurrent_mutex' + autoload :WindowedMutex, 'redis_mutex/windowed_mutex' + + include StandardMutex + include CumulativeMutex + include WindowedMutex + include ConcurrentMutex DEFAULT_EXPIRE = 10 LockError = Class.new(StandardError) @@ -18,14 +37,28 @@ class RedisMutex < RedisClassy def initialize(object, options={}) super(object.is_a?(String) || object.is_a?(Symbol) ? object : "#{object.class.name}:#{object.id}") - @block = options[:block] || 1 - @sleep = options[:sleep] || 0.1 - @expire = options[:expire] || DEFAULT_EXPIRE + @block = options[:block]&.to_f || 1 + @sleep = options[:sleep]&.to_f || 0.1 + @expire = options[:expire]&.to_i || DEFAULT_EXPIRE + @limit = options[:limit]&.to_i || 1 + @type = options[:type]&.to_sym || :concurrent + @unique_key = SecureRandom.uuid.to_s + raise ArgumentError, "Unknown type: #{@type}" unless %i[cumulative windowed concurrent].include?(@type) + end + + def type + case @type + when :cumulative then :cumulative + when :windowed then :windowed + when :concurrent + @limit == 1 ? :standard : :concurrent + end end def lock self.class.raise_assertion_error if block_given? @locking = false + cleanup_set if @block > 0 # Blocking mode @@ -38,40 +71,22 @@ def lock # Non-blocking mode @locking = try_lock end + + cleanup_set @locking end def try_lock - now = Time.now.to_f - @expires_at = now + @expire # Extend in each blocking loop - - begin - return true if setnx(@expires_at) # Success, the lock has been acquired - end until old_value = get # Repeat if unlocked before get - - return false if old_value.to_f > now # Check if the lock is still effective - - # The lock has expired but wasn't released... BAD! - return true if getset(@expires_at).to_f <= now # Success, we acquired the previously expired lock - return false # Dammit, it seems that someone else was even faster than us to remove the expired lock! + public_send("#{type}_try_lock") end # Returns true if resource is locked. Note that nil.to_f returns 0.0 - def locked? - get.to_f > Time.now.to_f + def locked?(now: Time.now.to_i, limit: @limit) + public_send("#{type}_locked?", now: now, limit: limit) end def unlock(force = false) - # Since it's possible that the operations in the critical section took a long time, - # we can't just simply release the lock. The unlock method checks if @expires_at - # remains the same, and do not release when the lock timestamp was overwritten. - - if get == @expires_at.to_s or force - # Redis#del with a single key returns '1' or nil - !!del - else - false - end + public_send("#{type}_unlock", force) end def with_lock @@ -93,23 +108,44 @@ def unlock!(force = false) unlock(force) or raise UnlockError, "failed to release lock #{key.inspect}" end + def cleanup_set(now: Time.now.to_i) + public_send("#{type}_cleanup_set", now: now) + end + + def key_count(now: Time.now.to_i) + public_send("#{type}_key_count", now: now) + end + class << self def sweep - return 0 if (all_keys = keys).empty? - + all_redis_mutex_keys = all_keys now = Time.now.to_f - values = mget(*all_keys) + total = 0 + total += standard_sweep(now, all_redis_mutex_keys[:standard]) + total += cumulative_sweep(now, all_redis_mutex_keys[:cumulative]) + total += windowed_sweep(now, all_redis_mutex_keys[:windowed]) + total += concurrent_sweep(now, all_redis_mutex_keys[:concurrent]) - expired_keys = all_keys.zip(values).select do |key, time| - time && time.to_f <= now - end + total + end - expired_keys.each do |key, _| - # Make extra sure that anyone haven't extended the lock - del(key) if getset(key, now + DEFAULT_EXPIRE).to_f <= now + def all_keys + return [] if (all_keys = scan_each.to_a).empty? + + all_keys.zip(mget(*all_keys)).each_with_object(Hash.new { |h, k| h[k] = [] }) do |hash, (key, value)| + if value.nil? + if key.end_with?(':cumulative_set') + hash[:cumulative] << key + elsif key.end_with?(':windowed_list') + hash[:windowed] << key + elsif key.end_with?(':concurrent_set') + hash[:concurrent] << key + end + else + hash[:standard] << [key, value] + end + hash end - - expired_keys.size end def lock(object, options = {}) diff --git a/lib/redis_mutex/concurrent_mutex.rb b/lib/redis_mutex/concurrent_mutex.rb new file mode 100644 index 0000000..1b3f7ad --- /dev/null +++ b/lib/redis_mutex/concurrent_mutex.rb @@ -0,0 +1,34 @@ +class RedisMutex < RedisClassy + module ConcurrentMutex + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + def cumulative_sweep(...) + concurrent_sweep(...) + end + end + + def concurrent_try_lock + cumulative_try_lock + end + def concurrent_locked?(...) + cumulative_locked?(...) + end + def concurrent_cleanup_set(...) + cumulative_cleanup_set(...) + end + def concurrent_key_count(...) + cumulative_key_count(...) + end + + def concurrent_unlock(_) + key_was = key + self.key = "#{key}:#{type}_set" + !!zrem(@unique_key) + ensure + self.key = key_was + end + end +end \ No newline at end of file diff --git a/lib/redis_mutex/cumulative_mutex.rb b/lib/redis_mutex/cumulative_mutex.rb new file mode 100644 index 0000000..a4bcd30 --- /dev/null +++ b/lib/redis_mutex/cumulative_mutex.rb @@ -0,0 +1,75 @@ +class RedisMutex < RedisClassy + module CumulativeMutex + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + def cumulative_sweep(now, all_keys) + return 0 if all_keys.nil? || all_keys.empty? + + multi do |pipeline| + all_keys.each do |redis_key| + pipeline.zremrangebyscore(redis_key, '-inf', "#{now.to_i - DEFAULT_EXPIRE})") + end + end + end + end + + def cumulative_try_lock + RedisMutex.with_lock(key, limit: 1, expire: @expire, block: 0, sleep: @sleep) do + now = Time.now.to_i + if locked?(now: now) + false + else + begin + key_was = key + self.key = "#{key}:#{type}_set" + zadd(now, @unique_key) + ensure + self.key = key_was + end + true + end + end + rescue RedisMutex::LockError + # If we can't get the lock, we can't get lock + false + end + + def cumulative_locked?(now: Time.now.to_i, limit: @limit) + key_count(now: now) >= limit + end + + def cumulative_unlock(force = false) + if force + !!unlink + else + false + end + end + + def cumulative_key_count(now: Time.now.to_i) + key_was = key + self.key = "#{key}:#{type}_set" + zcount(now - @expire, now) + rescue Redis::BaseError + 0 + ensure + self.key = key_was + end + + def cumulative_cleanup_set(now: Time.now.to_i) + key_was = key + self.key = "#{key_was}:#{type}_set" + # Any set members with a score lower than the current time minus the expire time are no longer needed + # This is to optimize the key_count too O(log(N)) and the cleanup which is O(log(N)+M) + # So cleanup should be run as often as possible + zremrangebyscore('-inf', "(#{now - @expire}") + rescue RedisMutex::LockError + nil + ensure + self.key = key_was + end + end +end \ No newline at end of file diff --git a/lib/redis_mutex/standard_mutex.rb b/lib/redis_mutex/standard_mutex.rb new file mode 100644 index 0000000..643ced8 --- /dev/null +++ b/lib/redis_mutex/standard_mutex.rb @@ -0,0 +1,66 @@ +class RedisMutex < RedisClassy + module StandardMutex + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + def standard_sweep(now, all_keys) + expired_keys = all_keys.select do |key, time| + time.to_f <= now + end + + expired_keys.each do |redis_key, _| + # Make extra sure that anyone haven't extended the lock + unlink(redis_key) if getset(redis_key, now + DEFAULT_EXPIRE).to_f <= now + end + + expired_keys.size + end + end + + def standard_try_lock + now = Time.now.to_f + @expires_at = now + @expire # Extend in each blocking loop + + begin + if setnx(@expires_at) + return true # Success, the lock has been acquired + end + end until old_value = get # Repeat if unlocked before get + + return false if old_value.to_f > now # Check if the lock is still effective + + # The lock has expired but wasn't released... BAD! + if getset(@expires_at).to_f <= now + return true # Success, we acquired the previously expired lock + end + return false # Dammit, it seems that someone else was even faster than us to remove the expired lock! + end + + def standard_locked?(...) + get.to_f > Time.now.to_f + end + + def standard_unlock(force = false) + # Since it's possible that the operations in the critical section took a long time, + # we can't just simply release the lock. The unlock method checks if @expires_at + # remains the same, and do not release when the lock timestamp was overwritten. + + if get == @expires_at.to_s or force + # Redis#unlink with a single key returns '1' or nil + !!unlink + else + false + end + end + + def standard_key_count(...) + exists? ? 1 : 0 + end + + def standard_cleanup_set(...) + nil + end + end +end \ No newline at end of file diff --git a/lib/redis_mutex/windowed_mutex.rb b/lib/redis_mutex/windowed_mutex.rb new file mode 100644 index 0000000..1954cfe --- /dev/null +++ b/lib/redis_mutex/windowed_mutex.rb @@ -0,0 +1,57 @@ +class RedisMutex < RedisClassy + module WindowedMutex + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + def windowed_sweep(...) + # windowed mutexes use expire to automatically remove keys + 0 + end + end + + def windowed_try_lock + RedisMutex.with_lock(key, limit: 1, expire: @expire, block: 0, sleep: @sleep) do + if locked? + false + else + begin + key_was = key + self.key = "#{key}:#{type}_list" + lpush(@unique_key) + expire(@expire, nx: true) # only set expire if it does not have one + ensure + self.key = key_was + end + true + end + end + rescue RedisMutex::LockError + # If we can't get the lock, we can't get lock + false + end + + def windowed_locked?(...) + cumulative_locked?(...) + end + + def windowed_unlock(...) + cumulative_unlock(...) + end + + def windowed_key_count(...) + key_was = key + self.key = "#{key}:#{type}_list" + llen + rescue Redis::BaseError + 0 + ensure + self.key = key_was + end + + def windowed_cleanup_set(...) + nil + end + end +end \ No newline at end of file From b913f7e9f2b6ff9d2367e229b6fc6e4f342d3c7a Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:29:38 +0100 Subject: [PATCH 02/20] fix specs and add a github action workflow (it's free for opensource --- .github/workflows/rspec.yml | 35 +++++++++++++++++++++++++++++ .vscode/settings.json | 3 +++ Rakefile | 6 ++++- lib/redis_mutex.rb | 2 +- lib/redis_mutex/concurrent_mutex.rb | 10 +++------ lib/redis_mutex/cumulative_mutex.rb | 22 +++--------------- lib/redis_mutex/macro.rb | 8 +++---- lib/redis_mutex/windowed_mutex.rb | 17 ++++---------- spec/spec_helper.rb | 10 ++++++--- 9 files changed, 65 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/rspec.yml create mode 100644 .vscode/settings.json diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml new file mode 100644 index 0000000..3e28297 --- /dev/null +++ b/.github/workflows/rspec.yml @@ -0,0 +1,35 @@ +name: RSpec + +on: + pull_request: + +jobs: + rspec: + name: RSpec + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ruby-version: [2.1, 2.2, 2.3.0, 2.7.0, 3.0, 3.1, 3.2, 3.3] + + services: + redis: + image: public.ecr.aws/docker/library/redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + + - name: Run rspec + run: bundle exec rspec diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..0a77011 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "editor.tabSize": 2 +} \ No newline at end of file diff --git a/Rakefile b/Rakefile index 1ed8b79..8b6c70e 100644 --- a/Rakefile +++ b/Rakefile @@ -10,5 +10,9 @@ task :default => :spec desc 'Flush the test database' task :flushdb do require 'redis' - Redis.new(:db => 15).flushdb + if ENV['ci'] == 'true' + Redis.new.flushdb + else + Redis.new(db: 15).flushdb + end end diff --git a/lib/redis_mutex.rb b/lib/redis_mutex.rb index 03c80a1..366eebc 100644 --- a/lib/redis_mutex.rb +++ b/lib/redis_mutex.rb @@ -132,7 +132,7 @@ def sweep def all_keys return [] if (all_keys = scan_each.to_a).empty? - all_keys.zip(mget(*all_keys)).each_with_object(Hash.new { |h, k| h[k] = [] }) do |hash, (key, value)| + all_keys.zip(mget(*all_keys)).each_with_object(Hash.new { |h, k| h[k] = [] }) do |(key, value), hash| if value.nil? if key.end_with?(':cumulative_set') hash[:cumulative] << key diff --git a/lib/redis_mutex/concurrent_mutex.rb b/lib/redis_mutex/concurrent_mutex.rb index 1b3f7ad..1a76e1c 100644 --- a/lib/redis_mutex/concurrent_mutex.rb +++ b/lib/redis_mutex/concurrent_mutex.rb @@ -5,8 +5,8 @@ def self.included(base) end module ClassMethods - def cumulative_sweep(...) - concurrent_sweep(...) + def concurrent_sweep(...) + cumulative_sweep(...) end end @@ -24,11 +24,7 @@ def concurrent_key_count(...) end def concurrent_unlock(_) - key_was = key - self.key = "#{key}:#{type}_set" - !!zrem(@unique_key) - ensure - self.key = key_was + !!redis.zrem("#{key}:#{type}_set", @unique_key) end end end \ No newline at end of file diff --git a/lib/redis_mutex/cumulative_mutex.rb b/lib/redis_mutex/cumulative_mutex.rb index a4bcd30..eed6d5e 100644 --- a/lib/redis_mutex/cumulative_mutex.rb +++ b/lib/redis_mutex/cumulative_mutex.rb @@ -22,13 +22,7 @@ def cumulative_try_lock if locked?(now: now) false else - begin - key_was = key - self.key = "#{key}:#{type}_set" - zadd(now, @unique_key) - ensure - self.key = key_was - end + redis.zadd("#{key}:#{type}_set", now, @unique_key) true end end @@ -50,26 +44,16 @@ def cumulative_unlock(force = false) end def cumulative_key_count(now: Time.now.to_i) - key_was = key - self.key = "#{key}:#{type}_set" - zcount(now - @expire, now) + redis.zcount("#{key}:#{type}_set", now - @expire, now) rescue Redis::BaseError 0 - ensure - self.key = key_was end def cumulative_cleanup_set(now: Time.now.to_i) - key_was = key - self.key = "#{key_was}:#{type}_set" # Any set members with a score lower than the current time minus the expire time are no longer needed # This is to optimize the key_count too O(log(N)) and the cleanup which is O(log(N)+M) # So cleanup should be run as often as possible - zremrangebyscore('-inf', "(#{now - @expire}") - rescue RedisMutex::LockError - nil - ensure - self.key = key_was + redis.zremrangebyscore("#{key}:#{type}_set", '-inf', "(#{now - @expire}") end end end \ No newline at end of file diff --git a/lib/redis_mutex/macro.rb b/lib/redis_mutex/macro.rb index f8bad94..be534bd 100644 --- a/lib/redis_mutex/macro.rb +++ b/lib/redis_mutex/macro.rb @@ -35,8 +35,8 @@ def method_added(target) raise ArgumentError, "You are trying to lock on unknown arguments: #{unknown_arguments.join(', ')}" end - define_method(with_method) do |*args| - named_arguments = Hash[target_argument_names.zip(args)] + define_method(with_method) do |*args, **kwargs| + named_arguments = kwargs.merge(Hash[target_argument_names.zip(args)].compact) arguments = mutex_arguments.map { |name| named_arguments.fetch(name) } key = format( "%s#%s:%s", @@ -46,10 +46,10 @@ def method_added(target) ) begin RedisMutex.with_lock(key, options) do - send(without_method, *args) + send(without_method, *args, **kwargs) end rescue RedisMutex::LockError - send(after_method, *args) if respond_to?(after_method) + send(after_method, *args, **kwargs) if respond_to?(after_method) end end diff --git a/lib/redis_mutex/windowed_mutex.rb b/lib/redis_mutex/windowed_mutex.rb index 1954cfe..1db2498 100644 --- a/lib/redis_mutex/windowed_mutex.rb +++ b/lib/redis_mutex/windowed_mutex.rb @@ -16,14 +16,9 @@ def windowed_try_lock if locked? false else - begin - key_was = key - self.key = "#{key}:#{type}_list" - lpush(@unique_key) - expire(@expire, nx: true) # only set expire if it does not have one - ensure - self.key = key_was - end + windowed_key = "#{key}:#{type}_list" + redis.lpush(windowed_key, @unique_key) + redis.expire(windowed_key, @expire, nx: true) # only set expire if it does not have one true end end @@ -41,13 +36,9 @@ def windowed_unlock(...) end def windowed_key_count(...) - key_was = key - self.key = "#{key}:#{type}_list" - llen + redis.llen("#{key}:#{type}_list") rescue Redis::BaseError 0 - ensure - self.key = key_was end def windowed_cleanup_set(...) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e17d5e7..e574758 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,9 +5,13 @@ require 'redis-mutex' RSpec.configure do |config| - # Use database 15 for testing so we don't accidentally step on you real data. - RedisClassy.redis = Redis.new(db: 15) + if ENV['ci'] == 'true' + RedisClassy.redis = Redis.new + else + # Use database 15 for testing so we don't accidentally step on you real data. + RedisClassy.redis = Redis.new(db: 15) + end unless RedisClassy.keys.empty? - abort '[ERROR]: Redis database 15 not empty! If you are sure, run "rake flushdb" beforehand.' + abort '[ERROR]: Redis database is not empty! If you are sure, run "rake flushdb" beforehand.' end end From 9d1e661a4c5095e5b3f1ac8a99fbb611c02d724a Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:31:46 +0100 Subject: [PATCH 03/20] forgot env var and add concurrency cancellation --- .github/workflows/rspec.yml | 7 +++++++ Rakefile | 2 +- spec/spec_helper.rb | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 3e28297..7beb2f7 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -3,6 +3,13 @@ name: RSpec on: pull_request: +env: + CI: true + +concurrency: + group: ${{ github.event.repository.name }}-${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + jobs: rspec: name: RSpec diff --git a/Rakefile b/Rakefile index 8b6c70e..45f1e3e 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ task :default => :spec desc 'Flush the test database' task :flushdb do require 'redis' - if ENV['ci'] == 'true' + if ENV['CI'] == 'true' Redis.new.flushdb else Redis.new(db: 15).flushdb diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e574758..bbb6c57 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,7 +5,7 @@ require 'redis-mutex' RSpec.configure do |config| - if ENV['ci'] == 'true' + if ENV['CI'] == 'true' RedisClassy.redis = Redis.new else # Use database 15 for testing so we don't accidentally step on you real data. From 1b170630098652b90bc8e57f14c55039422bef64 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:33:22 +0100 Subject: [PATCH 04/20] never remember if it's service name or localhost --- .github/workflows/rspec.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 7beb2f7..ec61d25 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -5,6 +5,7 @@ on: env: CI: true + REDIS_URL: redis://redis:6379/1 concurrency: group: ${{ github.event.repository.name }}-${{ github.workflow }}-${{ github.head_ref || github.ref_name }} From 6643830258fe992d5860072b35b3a6f3fc52d4d4 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:34:34 +0100 Subject: [PATCH 05/20] try localhost then ? --- .github/workflows/rspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index ec61d25..77436a5 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -5,7 +5,7 @@ on: env: CI: true - REDIS_URL: redis://redis:6379/1 + REDIS_URL: redis://localhost:6379/1 concurrency: group: ${{ github.event.repository.name }}-${{ github.workflow }}-${{ github.head_ref || github.ref_name }} From 25138e9b80cd1e88e26df82630a6d1e709bc8945 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:35:03 +0100 Subject: [PATCH 06/20] remove vscode settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 0a77011..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "editor.tabSize": 2 -} \ No newline at end of file From 0430da2df70228102b454da0a0d9a22acdfe0054 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:36:32 +0100 Subject: [PATCH 07/20] use docker redis ? --- .github/workflows/rspec.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 77436a5..99dd0f3 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -23,7 +23,9 @@ jobs: services: redis: - image: public.ecr.aws/docker/library/redis:7-alpine + # Docker Hub image + image: redis + # Set health checks to wait until redis has started options: >- --health-cmd "redis-cli ping" --health-interval 10s From e0286d8307a0533be3c8a15f8c4dcfb60de9d5d8 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:39:43 +0100 Subject: [PATCH 08/20] need to add ports --- .github/workflows/rspec.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 99dd0f3..148f852 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -5,7 +5,7 @@ on: env: CI: true - REDIS_URL: redis://localhost:6379/1 + REDIS_URL: redis://redis:6379/1 concurrency: group: ${{ github.event.repository.name }}-${{ github.workflow }}-${{ github.head_ref || github.ref_name }} @@ -23,9 +23,9 @@ jobs: services: redis: - # Docker Hub image - image: redis - # Set health checks to wait until redis has started + image: public.ecr.aws/docker/library/redis:7-alpine + ports: + - "6379:6379" options: >- --health-cmd "redis-cli ping" --health-interval 10s From 4c9d7d6c14a8d481875e7028322c66a1418064ff Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:40:40 +0100 Subject: [PATCH 09/20] localhost again --- .github/workflows/rspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 148f852..db09c2d 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -5,7 +5,7 @@ on: env: CI: true - REDIS_URL: redis://redis:6379/1 + REDIS_URL: redis://localhost:6379/1 concurrency: group: ${{ github.event.repository.name }}-${{ github.workflow }}-${{ github.head_ref || github.ref_name }} From 5bcffff5ed7be5903a48d2c11562dba9f6d1e47a Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:45:41 +0100 Subject: [PATCH 10/20] ellipsis is ruby 3+ and safe navigation is ruby 2.3+ --- lib/redis_mutex.rb | 10 +++++----- lib/redis_mutex/concurrent_mutex.rb | 16 ++++++++-------- lib/redis_mutex/standard_mutex.rb | 6 +++--- lib/redis_mutex/windowed_mutex.rb | 14 +++++++------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/redis_mutex.rb b/lib/redis_mutex.rb index 366eebc..ae9f5c8 100644 --- a/lib/redis_mutex.rb +++ b/lib/redis_mutex.rb @@ -37,11 +37,11 @@ class RedisMutex < RedisClassy def initialize(object, options={}) super(object.is_a?(String) || object.is_a?(Symbol) ? object : "#{object.class.name}:#{object.id}") - @block = options[:block]&.to_f || 1 - @sleep = options[:sleep]&.to_f || 0.1 - @expire = options[:expire]&.to_i || DEFAULT_EXPIRE - @limit = options[:limit]&.to_i || 1 - @type = options[:type]&.to_sym || :concurrent + @block = (options[:block] || 1).to_f + @sleep = (options[:sleep] || 0.1).to_f + @expire = (options[:expire] || DEFAULT_EXPIRE).to_i + @limit = (options[:limit] || 1).to_i + @type = (options[:type] || :concurrent).to_sym @unique_key = SecureRandom.uuid.to_s raise ArgumentError, "Unknown type: #{@type}" unless %i[cumulative windowed concurrent].include?(@type) end diff --git a/lib/redis_mutex/concurrent_mutex.rb b/lib/redis_mutex/concurrent_mutex.rb index 1a76e1c..4fbdb65 100644 --- a/lib/redis_mutex/concurrent_mutex.rb +++ b/lib/redis_mutex/concurrent_mutex.rb @@ -5,22 +5,22 @@ def self.included(base) end module ClassMethods - def concurrent_sweep(...) - cumulative_sweep(...) + def concurrent_sweep(*args, **kwargs) + cumulative_sweep(*args, **kwargs) end end def concurrent_try_lock cumulative_try_lock end - def concurrent_locked?(...) - cumulative_locked?(...) + def concurrent_locked?(*args, **kwargs) + cumulative_locked?(*args, **kwargs) end - def concurrent_cleanup_set(...) - cumulative_cleanup_set(...) + def concurrent_cleanup_set(*args, **kwargs) + cumulative_cleanup_set(*args, **kwargs) end - def concurrent_key_count(...) - cumulative_key_count(...) + def concurrent_key_count(*args, **kwargs) + cumulative_key_count(*args, **kwargs) end def concurrent_unlock(_) diff --git a/lib/redis_mutex/standard_mutex.rb b/lib/redis_mutex/standard_mutex.rb index 643ced8..922268a 100644 --- a/lib/redis_mutex/standard_mutex.rb +++ b/lib/redis_mutex/standard_mutex.rb @@ -38,7 +38,7 @@ def standard_try_lock return false # Dammit, it seems that someone else was even faster than us to remove the expired lock! end - def standard_locked?(...) + def standard_locked?(*args, **kwargs) get.to_f > Time.now.to_f end @@ -55,11 +55,11 @@ def standard_unlock(force = false) end end - def standard_key_count(...) + def standard_key_count(*args, **kwargs) exists? ? 1 : 0 end - def standard_cleanup_set(...) + def standard_cleanup_set(*args, **kwargs) nil end end diff --git a/lib/redis_mutex/windowed_mutex.rb b/lib/redis_mutex/windowed_mutex.rb index 1db2498..a0e4561 100644 --- a/lib/redis_mutex/windowed_mutex.rb +++ b/lib/redis_mutex/windowed_mutex.rb @@ -5,7 +5,7 @@ def self.included(base) end module ClassMethods - def windowed_sweep(...) + def windowed_sweep(*args, **kwargs) # windowed mutexes use expire to automatically remove keys 0 end @@ -27,21 +27,21 @@ def windowed_try_lock false end - def windowed_locked?(...) - cumulative_locked?(...) + def windowed_locked?(*args, **kwargs) + cumulative_locked?(*args, **kwargs) end - def windowed_unlock(...) - cumulative_unlock(...) + def windowed_unlock(*args, **kwargs) + cumulative_unlock(*args, **kwargs) end - def windowed_key_count(...) + def windowed_key_count(*args, **kwargs) redis.llen("#{key}:#{type}_list") rescue Redis::BaseError 0 end - def windowed_cleanup_set(...) + def windowed_cleanup_set(*args, **kwargs) nil end end From f9a54d85e358390d8e7e3f204042319b07c4c020 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 01:57:15 +0100 Subject: [PATCH 11/20] go back in time and give up on kwargs --- lib/redis_mutex.rb | 12 ++++++------ lib/redis_mutex/concurrent_mutex.rb | 16 ++++++++-------- lib/redis_mutex/cumulative_mutex.rb | 10 ++++++---- lib/redis_mutex/standard_mutex.rb | 6 +++--- lib/redis_mutex/windowed_mutex.rb | 14 +++++++------- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/lib/redis_mutex.rb b/lib/redis_mutex.rb index ae9f5c8..b10aefe 100644 --- a/lib/redis_mutex.rb +++ b/lib/redis_mutex.rb @@ -81,8 +81,8 @@ def try_lock end # Returns true if resource is locked. Note that nil.to_f returns 0.0 - def locked?(now: Time.now.to_i, limit: @limit) - public_send("#{type}_locked?", now: now, limit: limit) + def locked?(options = {}) + public_send("#{type}_locked?", options) end def unlock(force = false) @@ -108,12 +108,12 @@ def unlock!(force = false) unlock(force) or raise UnlockError, "failed to release lock #{key.inspect}" end - def cleanup_set(now: Time.now.to_i) - public_send("#{type}_cleanup_set", now: now) + def cleanup_set(options = {}) + public_send("#{type}_cleanup_set", options) end - def key_count(now: Time.now.to_i) - public_send("#{type}_key_count", now: now) + def key_count(options = {}) + public_send("#{type}_key_count", options) end class << self diff --git a/lib/redis_mutex/concurrent_mutex.rb b/lib/redis_mutex/concurrent_mutex.rb index 4fbdb65..5fc45ef 100644 --- a/lib/redis_mutex/concurrent_mutex.rb +++ b/lib/redis_mutex/concurrent_mutex.rb @@ -5,22 +5,22 @@ def self.included(base) end module ClassMethods - def concurrent_sweep(*args, **kwargs) - cumulative_sweep(*args, **kwargs) + def concurrent_sweep(now, all_keys) + cumulative_sweep(now, all_keys) end end def concurrent_try_lock cumulative_try_lock end - def concurrent_locked?(*args, **kwargs) - cumulative_locked?(*args, **kwargs) + def concurrent_locked?(options = {}) + cumulative_locked?(options) end - def concurrent_cleanup_set(*args, **kwargs) - cumulative_cleanup_set(*args, **kwargs) + def concurrent_cleanup_set(options = {}) + cumulative_cleanup_set(options) end - def concurrent_key_count(*args, **kwargs) - cumulative_key_count(*args, **kwargs) + def concurrent_key_count(options = {}) + cumulative_key_count(options) end def concurrent_unlock(_) diff --git a/lib/redis_mutex/cumulative_mutex.rb b/lib/redis_mutex/cumulative_mutex.rb index eed6d5e..57519f9 100644 --- a/lib/redis_mutex/cumulative_mutex.rb +++ b/lib/redis_mutex/cumulative_mutex.rb @@ -31,8 +31,8 @@ def cumulative_try_lock false end - def cumulative_locked?(now: Time.now.to_i, limit: @limit) - key_count(now: now) >= limit + def cumulative_locked?(options = {}) + key_count(now: options[:now] || Time.now.to_i) >= options[:limit] || @limit end def cumulative_unlock(force = false) @@ -43,13 +43,15 @@ def cumulative_unlock(force = false) end end - def cumulative_key_count(now: Time.now.to_i) + def cumulative_key_count(options = {}) + now = options[:now] || Time.now.to_i redis.zcount("#{key}:#{type}_set", now - @expire, now) rescue Redis::BaseError 0 end - def cumulative_cleanup_set(now: Time.now.to_i) + def cumulative_cleanup_set(options = {}) + now = options[:now] || Time.now.to_i # Any set members with a score lower than the current time minus the expire time are no longer needed # This is to optimize the key_count too O(log(N)) and the cleanup which is O(log(N)+M) # So cleanup should be run as often as possible diff --git a/lib/redis_mutex/standard_mutex.rb b/lib/redis_mutex/standard_mutex.rb index 922268a..7e3e2b0 100644 --- a/lib/redis_mutex/standard_mutex.rb +++ b/lib/redis_mutex/standard_mutex.rb @@ -38,7 +38,7 @@ def standard_try_lock return false # Dammit, it seems that someone else was even faster than us to remove the expired lock! end - def standard_locked?(*args, **kwargs) + def standard_locked?(_ = {}) get.to_f > Time.now.to_f end @@ -55,11 +55,11 @@ def standard_unlock(force = false) end end - def standard_key_count(*args, **kwargs) + def standard_key_count(_ = {}) exists? ? 1 : 0 end - def standard_cleanup_set(*args, **kwargs) + def standard_cleanup_set(_ = {}) nil end end diff --git a/lib/redis_mutex/windowed_mutex.rb b/lib/redis_mutex/windowed_mutex.rb index a0e4561..a337706 100644 --- a/lib/redis_mutex/windowed_mutex.rb +++ b/lib/redis_mutex/windowed_mutex.rb @@ -5,7 +5,7 @@ def self.included(base) end module ClassMethods - def windowed_sweep(*args, **kwargs) + def windowed_sweep(_now, _all_keys) # windowed mutexes use expire to automatically remove keys 0 end @@ -27,21 +27,21 @@ def windowed_try_lock false end - def windowed_locked?(*args, **kwargs) - cumulative_locked?(*args, **kwargs) + def windowed_locked?(options = {}) + cumulative_locked?(options) end - def windowed_unlock(*args, **kwargs) - cumulative_unlock(*args, **kwargs) + def windowed_unlock(force = false) + cumulative_unlock(force) end - def windowed_key_count(*args, **kwargs) + def windowed_key_count(_ = {}) redis.llen("#{key}:#{type}_list") rescue Redis::BaseError 0 end - def windowed_cleanup_set(*args, **kwargs) + def windowed_cleanup_set(_ = {}) nil end end From 7ab5f1742ec1efa525b671b66b05ed79ea420e42 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 02:05:52 +0100 Subject: [PATCH 12/20] add a delete_key method to prioritize unlink --- lib/redis_mutex.rb | 16 ++++++++++++++++ lib/redis_mutex/cumulative_mutex.rb | 2 +- lib/redis_mutex/standard_mutex.rb | 6 +++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/redis_mutex.rb b/lib/redis_mutex.rb index b10aefe..4444b81 100644 --- a/lib/redis_mutex.rb +++ b/lib/redis_mutex.rb @@ -116,6 +116,14 @@ def key_count(options = {}) public_send("#{type}_key_count", options) end + def delete_key(target_key = nil) + if Redis::Namespace::VERSION.to_i < 2 + target_key.present? ? redis.del(redis_key) : del + else + target_key.present? ? redis.unlink(redis_key) : unlink + end + end + class << self def sweep all_redis_mutex_keys = all_keys @@ -164,5 +172,13 @@ def with_lock(object, options = {}, &block) def raise_assertion_error raise AssertionError, 'block syntax has been removed from #lock, use #with_lock instead' end + + def delete_key(target_key = nil) + if Redis::Namespace::VERSION.to_i < 2 + target_key.present? ? redis.del(redis_key) : del + else + target_key.present? ? redis.unlink(redis_key) : unlink + end + end end end diff --git a/lib/redis_mutex/cumulative_mutex.rb b/lib/redis_mutex/cumulative_mutex.rb index 57519f9..d851c19 100644 --- a/lib/redis_mutex/cumulative_mutex.rb +++ b/lib/redis_mutex/cumulative_mutex.rb @@ -37,7 +37,7 @@ def cumulative_locked?(options = {}) def cumulative_unlock(force = false) if force - !!unlink + !!delete_key else false end diff --git a/lib/redis_mutex/standard_mutex.rb b/lib/redis_mutex/standard_mutex.rb index 7e3e2b0..7da0e61 100644 --- a/lib/redis_mutex/standard_mutex.rb +++ b/lib/redis_mutex/standard_mutex.rb @@ -12,7 +12,7 @@ def standard_sweep(now, all_keys) expired_keys.each do |redis_key, _| # Make extra sure that anyone haven't extended the lock - unlink(redis_key) if getset(redis_key, now + DEFAULT_EXPIRE).to_f <= now + delete_key(redis_key) if getset(redis_key, now + DEFAULT_EXPIRE).to_f <= now end expired_keys.size @@ -48,8 +48,8 @@ def standard_unlock(force = false) # remains the same, and do not release when the lock timestamp was overwritten. if get == @expires_at.to_s or force - # Redis#unlink with a single key returns '1' or nil - !!unlink + # Redis#unlink or Redis#del with a single key returns '1' or nil + !!delete_key else false end From 9f6a64f0a736084f8ceeb44a95ea6d4d71dad7b3 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 02:07:35 +0100 Subject: [PATCH 13/20] present? is active_support --- lib/redis_mutex.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/redis_mutex.rb b/lib/redis_mutex.rb index 4444b81..5ecc347 100644 --- a/lib/redis_mutex.rb +++ b/lib/redis_mutex.rb @@ -118,9 +118,9 @@ def key_count(options = {}) def delete_key(target_key = nil) if Redis::Namespace::VERSION.to_i < 2 - target_key.present? ? redis.del(redis_key) : del + target_key ? redis.del(target_key) : del else - target_key.present? ? redis.unlink(redis_key) : unlink + target_key ? redis.unlink(target_key) : unlink end end @@ -175,9 +175,9 @@ def raise_assertion_error def delete_key(target_key = nil) if Redis::Namespace::VERSION.to_i < 2 - target_key.present? ? redis.del(redis_key) : del + target_key ? redis.del(target_key) : del else - target_key.present? ? redis.unlink(redis_key) : unlink + target_key ? redis.unlink(target_key) : unlink end end end From 9bf73a06dbbf26696564298a51bf8b3826f87e8e Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 02:13:56 +0100 Subject: [PATCH 14/20] can't compact in older versions of ruby --- lib/redis_mutex/macro.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/redis_mutex/macro.rb b/lib/redis_mutex/macro.rb index be534bd..d55a6a7 100644 --- a/lib/redis_mutex/macro.rb +++ b/lib/redis_mutex/macro.rb @@ -36,7 +36,9 @@ def method_added(target) end define_method(with_method) do |*args, **kwargs| - named_arguments = kwargs.merge(Hash[target_argument_names.zip(args)].compact) + named_arguments = kwargs.merge( + Hash[target_argument_names.zip(args)].reject { |_, value| value.nil? } + ) arguments = mutex_arguments.map { |name| named_arguments.fetch(name) } key = format( "%s#%s:%s", From 09340c7fc0d57a62ff7150b4174103cd81205b27 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 08:26:56 +0100 Subject: [PATCH 15/20] added a devcontainer to help debug old ruby version (maybe should consider supporting ruby 3+ only ?) --- .devcontainer/.dockerignore | 1 + .devcontainer/Dockerfile | 27 +++++++++++++++ .devcontainer/devcontainer.json | 25 ++++++++++++++ .devcontainer/docker-compose.yml | 58 ++++++++++++++++++++++++++++++++ .gitignore | 1 + Gemfile | 2 +- lib/redis_mutex/macro.rb | 52 +++++++++++++++++++--------- spec/redis_mutex_macro_spec.rb | 2 +- 8 files changed, 150 insertions(+), 18 deletions(-) create mode 100644 .devcontainer/.dockerignore create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml diff --git a/.devcontainer/.dockerignore b/.devcontainer/.dockerignore new file mode 100644 index 0000000..68feb7d --- /dev/null +++ b/.devcontainer/.dockerignore @@ -0,0 +1 @@ +Gemfile.lock \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..1fac9e0 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,27 @@ +# FROM ruby:2.3.0-alpine + +# RUN apk add --no-cache \ +# bash \ +# git \ +# vim + +FROM debian:bullseye + +SHELL ["/bin/bash","-l","-c"] + +# Install dependencies +RUN apt-get update +RUN apt-get install ghostscript shared-mime-info openssl curl gnupg2 dirmngr git-core libcurl4-openssl-dev software-properties-common zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libffi-dev libpq-dev libmagickcore-6.q16-dev -y + +# Install rbenv & ruby-build +RUN git clone https://github.com/rbenv/rbenv.git ~/.rbenv +RUN echo 'export PATH="~/.rbenv/bin:$PATH"' >> ~/.bashrc +RUN echo 'eval "$(rbenv init -)"' >> ~/.bashrc + +RUN git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +RUN echo 'export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"' >> ~/.bashrc + +# Install ruby +RUN rbenv rehash +RUN rbenv install 2.3.0 +RUN rbenv global 2.3.0 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..184d42d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,25 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/docker-existing-docker-compose +// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml. +{ + "name": "Redis Mutex", + + // Update the 'dockerComposeFile' list if you have more compose files or use different names. + // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make. + "dockerComposeFile": "./docker-compose.yml", + + // The 'service' property is the name of the service for the container that VS Code should + // use. Update this value and .devcontainer/docker-compose.yml to the real service name. + "service": "app", + + // The optional 'workspaceFolder' property is the path VS Code should open by default when + // connected. This is typically a file mount in .devcontainer/docker-compose.yml + "workspaceFolder": "/app", + "shutdownAction": "stopCompose", + + "mounts": [ + "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,consistency=cached" + ], + + "postAttachCommand": "git config --global --add safe.directory /app", +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..ddd6e10 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,58 @@ +version: '3' +name: redis-mutex +networks: + redis-mutex: + name: redis-mutex + external: false +volumes: + bundle_cache: + name: 'redis-mutex-bundle-cache' + vendor_cache: + name: 'redis-mutex-vendor-cache' + redis_data: + name: 'redis-mutex-redis-data' + redis_shared_data: + name: 'redis-mutex-redis-data' +services: + redis: + image: redis:7-alpine + command: redis-server + container_name: redis-mutex-redis + volumes: + - 'redis_shared_data:/var/shared/redis:delegated' + - 'redis_data:/data:delegated' + networks: + - redis-mutex + expose: + - 6379 + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + interval: 10s + timeout: 5s + retries: 2 + start_period: 10s + restart: unless-stopped + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + image: redis-mutex + container_name: redis-mutex + pull_policy: build + volumes: + - '..:/app:cached' + - vendor_cache:/app/vendor/cache:delegated + - bundle_cache:/app/vendor/bundle:delegated + networks: + - redis-mutex + stdin_open: true + tty: true + environment: + BUNDLE_PATH: /app/vendor/bundle + REDIS_URL: redis://redis:6379/1 + CI: true + entrypoint: ['/bin/bash', '-l', '-c'] + command: '"tail -f /dev/null"' + depends_on: + redis: + condition: service_healthy diff --git a/.gitignore b/.gitignore index 8f0d248..df45fab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ pkg Gemfile.lock +vendor \ No newline at end of file diff --git a/Gemfile b/Gemfile index d65e2a6..8926307 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,3 @@ source 'http://rubygems.org' -gemspec +gemspec \ No newline at end of file diff --git a/lib/redis_mutex/macro.rb b/lib/redis_mutex/macro.rb index d55a6a7..c677118 100644 --- a/lib/redis_mutex/macro.rb +++ b/lib/redis_mutex/macro.rb @@ -35,23 +35,43 @@ def method_added(target) raise ArgumentError, "You are trying to lock on unknown arguments: #{unknown_arguments.join(', ')}" end - define_method(with_method) do |*args, **kwargs| - named_arguments = kwargs.merge( - Hash[target_argument_names.zip(args)].reject { |_, value| value.nil? } - ) - arguments = mutex_arguments.map { |name| named_arguments.fetch(name) } - key = format( - "%s#%s:%s", - class: self.class.name, - target: target, - arguments: arguments.join(":") - ) - begin - RedisMutex.with_lock(key, options) do - send(without_method, *args, **kwargs) + if RUBY_VERSION.to_f > 2.3 + define_method(with_method) do |*args, **kwargs| + named_arguments = kwargs.merge( + Hash[target_argument_names.zip(args)].reject { |_, value| value.nil? } + ) + arguments = mutex_arguments.map { |name| named_arguments.fetch(name) } + key = format( + "%s#%s:%s", + class: self.class.name, + target: target, + arguments: arguments.join(":") + ) + begin + RedisMutex.with_lock(key, options) do + send(without_method, *args, **kwargs) + end + rescue RedisMutex::LockError + send(after_method, *args, **kwargs) if respond_to?(after_method) + end + end + else + define_method(with_method) do |*args| + named_arguments = Hash[target_argument_names.zip(args)] + arguments = mutex_arguments.map { |name| named_arguments.fetch(name) } + key = format( + "%s#%s:%s", + class: self.class.name, + target: target, + arguments: arguments.join(":") + ) + begin + RedisMutex.with_lock(key, options) do + send(without_method, *args) + end + rescue RedisMutex::LockError + send(after_method, *args) if respond_to?(after_method) end - rescue RedisMutex::LockError - send(after_method, *args, **kwargs) if respond_to?(after_method) end end diff --git a/spec/redis_mutex_macro_spec.rb b/spec/redis_mutex_macro_spec.rb index 09e1661..306ef06 100644 --- a/spec/redis_mutex_macro_spec.rb +++ b/spec/redis_mutex_macro_spec.rb @@ -2,8 +2,8 @@ class C include RedisMutex::Macro - auto_mutex :run_singularly, :block => 0, :after_failure => lambda {|id| return "failure: #{id}" } + auto_mutex :run_singularly, :block => 0, :after_failure => lambda {|id| return "failure: #{id}" } def run_singularly(id) sleep 0.1 return "success: #{id}" From 064a2a77774c28cd18de2bdb07765185367d2f4d Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 09:08:30 +0100 Subject: [PATCH 16/20] add a Dockerfile for each matrixed version of the gem for easier debugging also remove rake dependency --- .devcontainer/.dockerignore | 3 ++- .devcontainer/Dockerfile | 28 +++------------------------- .devcontainer/Dockerfile.2.1 | 23 +++++++++++++++++++++++ .devcontainer/Dockerfile.2.2 | 23 +++++++++++++++++++++++ .devcontainer/Dockerfile.2.3.0 | 23 +++++++++++++++++++++++ .devcontainer/Dockerfile.2.7.0 | 5 +++++ .devcontainer/Dockerfile.3.0 | 5 +++++ .devcontainer/Dockerfile.3.1 | 5 +++++ .devcontainer/Dockerfile.3.2 | 5 +++++ .devcontainer/Dockerfile.3.3 | 5 +++++ .devcontainer/Dockerfile.3.4 | 5 +++++ .devcontainer/docker-compose.yml | 6 ------ .github/workflows/rspec.yml | 2 +- .travis.yml | 13 ------------- Rakefile | 16 ---------------- redis-mutex.gemspec | 3 --- spec/spec_helper.rb | 4 ++++ 17 files changed, 109 insertions(+), 65 deletions(-) create mode 100644 .devcontainer/Dockerfile.2.1 create mode 100644 .devcontainer/Dockerfile.2.2 create mode 100644 .devcontainer/Dockerfile.2.3.0 create mode 100644 .devcontainer/Dockerfile.2.7.0 create mode 100644 .devcontainer/Dockerfile.3.0 create mode 100644 .devcontainer/Dockerfile.3.1 create mode 100644 .devcontainer/Dockerfile.3.2 create mode 100644 .devcontainer/Dockerfile.3.3 create mode 100644 .devcontainer/Dockerfile.3.4 delete mode 100644 .travis.yml diff --git a/.devcontainer/.dockerignore b/.devcontainer/.dockerignore index 68feb7d..9bf06c0 100644 --- a/.devcontainer/.dockerignore +++ b/.devcontainer/.dockerignore @@ -1 +1,2 @@ -Gemfile.lock \ No newline at end of file +../Gemfile.lock +../vendor \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 1fac9e0..553e9cf 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,27 +1,5 @@ -# FROM ruby:2.3.0-alpine +FROM ruby:latest -# RUN apk add --no-cache \ -# bash \ -# git \ -# vim +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* -FROM debian:bullseye - -SHELL ["/bin/bash","-l","-c"] - -# Install dependencies -RUN apt-get update -RUN apt-get install ghostscript shared-mime-info openssl curl gnupg2 dirmngr git-core libcurl4-openssl-dev software-properties-common zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libffi-dev libpq-dev libmagickcore-6.q16-dev -y - -# Install rbenv & ruby-build -RUN git clone https://github.com/rbenv/rbenv.git ~/.rbenv -RUN echo 'export PATH="~/.rbenv/bin:$PATH"' >> ~/.bashrc -RUN echo 'eval "$(rbenv init -)"' >> ~/.bashrc - -RUN git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -RUN echo 'export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"' >> ~/.bashrc - -# Install ruby -RUN rbenv rehash -RUN rbenv install 2.3.0 -RUN rbenv global 2.3.0 \ No newline at end of file +RUN gem install bundler diff --git a/.devcontainer/Dockerfile.2.1 b/.devcontainer/Dockerfile.2.1 new file mode 100644 index 0000000..e2a75fa --- /dev/null +++ b/.devcontainer/Dockerfile.2.1 @@ -0,0 +1,23 @@ +# Need to build using rbenv because these are very old ruby verions and their images need rebuilding to work anyways +FROM debian:bullseye + +SHELL ["/bin/bash","-l","-c"] + +# Install dependencies +RUN apt-get update +RUN apt-get install ghostscript shared-mime-info openssl curl gnupg2 dirmngr git-core libcurl4-openssl-dev software-properties-common zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libffi-dev libpq-dev libmagickcore-6.q16-dev -y + +# Install rbenv & ruby-build +RUN git clone https://github.com/rbenv/rbenv.git ~/.rbenv +RUN echo 'export PATH="~/.rbenv/bin:$PATH"' >> ~/.bashrc +RUN echo 'eval "$(rbenv init -)"' >> ~/.bashrc + +RUN git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +RUN echo 'export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"' >> ~/.bashrc + +# Install ruby +RUN rbenv rehash +RUN rbenv install 2.1.10 +RUN rbenv global 2.1.10 + +RUN gem install bundler -v '< 2' diff --git a/.devcontainer/Dockerfile.2.2 b/.devcontainer/Dockerfile.2.2 new file mode 100644 index 0000000..a7e08da --- /dev/null +++ b/.devcontainer/Dockerfile.2.2 @@ -0,0 +1,23 @@ +# Need to build using rbenv because these are very old ruby verions and their images need rebuilding to work anyways +FROM debian:bullseye + +SHELL ["/bin/bash","-l","-c"] + +# Install dependencies +RUN apt-get update +RUN apt-get install ghostscript shared-mime-info openssl curl gnupg2 dirmngr git-core libcurl4-openssl-dev software-properties-common zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libffi-dev libpq-dev libmagickcore-6.q16-dev -y + +# Install rbenv & ruby-build +RUN git clone https://github.com/rbenv/rbenv.git ~/.rbenv +RUN echo 'export PATH="~/.rbenv/bin:$PATH"' >> ~/.bashrc +RUN echo 'eval "$(rbenv init -)"' >> ~/.bashrc + +RUN git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +RUN echo 'export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"' >> ~/.bashrc + +# Install ruby +RUN rbenv rehash +RUN rbenv install 2.2.10 +RUN rbenv global 2.2.10 + +RUN gem install bundler -v '< 2' diff --git a/.devcontainer/Dockerfile.2.3.0 b/.devcontainer/Dockerfile.2.3.0 new file mode 100644 index 0000000..36b7e16 --- /dev/null +++ b/.devcontainer/Dockerfile.2.3.0 @@ -0,0 +1,23 @@ +# Need to build using rbenv because these are very old ruby verions and their images need rebuilding to work anyways +FROM debian:bullseye + +SHELL ["/bin/bash","-l","-c"] + +# Install dependencies +RUN apt-get update +RUN apt-get install ghostscript shared-mime-info openssl curl gnupg2 dirmngr git-core libcurl4-openssl-dev software-properties-common zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libffi-dev libpq-dev libmagickcore-6.q16-dev -y + +# Install rbenv & ruby-build +RUN git clone https://github.com/rbenv/rbenv.git ~/.rbenv +RUN echo 'export PATH="~/.rbenv/bin:$PATH"' >> ~/.bashrc +RUN echo 'eval "$(rbenv init -)"' >> ~/.bashrc + +RUN git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +RUN echo 'export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH"' >> ~/.bashrc + +# Install ruby +RUN rbenv rehash +RUN rbenv install 2.3.0 +RUN rbenv global 2.3.0 + +RUN gem install bundler -v '< 2' diff --git a/.devcontainer/Dockerfile.2.7.0 b/.devcontainer/Dockerfile.2.7.0 new file mode 100644 index 0000000..9ae5ea6 --- /dev/null +++ b/.devcontainer/Dockerfile.2.7.0 @@ -0,0 +1,5 @@ +FROM ruby:2.7.0 + +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* + +RUN gem install bundler diff --git a/.devcontainer/Dockerfile.3.0 b/.devcontainer/Dockerfile.3.0 new file mode 100644 index 0000000..83cf9f8 --- /dev/null +++ b/.devcontainer/Dockerfile.3.0 @@ -0,0 +1,5 @@ +FROM ruby:3.0 + +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* + +RUN gem install bundler diff --git a/.devcontainer/Dockerfile.3.1 b/.devcontainer/Dockerfile.3.1 new file mode 100644 index 0000000..18c4929 --- /dev/null +++ b/.devcontainer/Dockerfile.3.1 @@ -0,0 +1,5 @@ +FROM ruby:3.1 + +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* + +RUN gem install bundler diff --git a/.devcontainer/Dockerfile.3.2 b/.devcontainer/Dockerfile.3.2 new file mode 100644 index 0000000..36de053 --- /dev/null +++ b/.devcontainer/Dockerfile.3.2 @@ -0,0 +1,5 @@ +FROM ruby:3.2 + +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* + +RUN gem install bundler diff --git a/.devcontainer/Dockerfile.3.3 b/.devcontainer/Dockerfile.3.3 new file mode 100644 index 0000000..493f07e --- /dev/null +++ b/.devcontainer/Dockerfile.3.3 @@ -0,0 +1,5 @@ +FROM ruby:3.3 + +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* + +RUN gem install bundler diff --git a/.devcontainer/Dockerfile.3.4 b/.devcontainer/Dockerfile.3.4 new file mode 100644 index 0000000..85a9594 --- /dev/null +++ b/.devcontainer/Dockerfile.3.4 @@ -0,0 +1,5 @@ +FROM ruby:3.4 + +RUN apt-get update && apt-get install -y bash git vim && rm -rf /var/lib/apt/lists/* + +RUN gem install bundler diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index ddd6e10..05df563 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -5,10 +5,6 @@ networks: name: redis-mutex external: false volumes: - bundle_cache: - name: 'redis-mutex-bundle-cache' - vendor_cache: - name: 'redis-mutex-vendor-cache' redis_data: name: 'redis-mutex-redis-data' redis_shared_data: @@ -41,8 +37,6 @@ services: pull_policy: build volumes: - '..:/app:cached' - - vendor_cache:/app/vendor/cache:delegated - - bundle_cache:/app/vendor/bundle:delegated networks: - redis-mutex stdin_open: true diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index db09c2d..e431af9 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: [2.1, 2.2, 2.3.0, 2.7.0, 3.0, 3.1, 3.2, 3.3] + ruby-version: [2.1, 2.2, 2.3.0, 2.7.0, 3.0, 3.1, 3.2, 3.3, 3.4] services: redis: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 94cc563..0000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: ruby -cache: bundler -services: - - redis-server -before_install: - - gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true - - gem install bundler -v '< 2' -rvm: - - 2.1 - - 2.2 - - 2.3.0 - - 2.7.0 -# - ruby-head diff --git a/Rakefile b/Rakefile index 45f1e3e..f57ae68 100644 --- a/Rakefile +++ b/Rakefile @@ -1,18 +1,2 @@ #!/usr/bin/env rake require "bundler/gem_tasks" - -# RSpec -require 'rspec/core/rake_task' -RSpec::Core::RakeTask.new('spec') -task :default => :spec - -# Custom Tasks -desc 'Flush the test database' -task :flushdb do - require 'redis' - if ENV['CI'] == 'true' - Redis.new.flushdb - else - Redis.new(db: 15).flushdb - end -end diff --git a/redis-mutex.gemspec b/redis-mutex.gemspec index 8cd7cff..7c7f40d 100644 --- a/redis-mutex.gemspec +++ b/redis-mutex.gemspec @@ -17,7 +17,4 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "redis-classy", "~> 2.0" gem.add_development_dependency "rspec" gem.add_development_dependency "bundler" - - # For Travis - gem.add_development_dependency "rake" end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index bbb6c57..b9abf70 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -14,4 +14,8 @@ unless RedisClassy.keys.empty? abort '[ERROR]: Redis database is not empty! If you are sure, run "rake flushdb" beforehand.' end + + config.before(:each) do + Redis.new.flushdb + end end From b9a09423b358e84c3648c907f05c544d43b6dc9c Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 08:13:48 +0000 Subject: [PATCH 17/20] actually dockerignore has no effect on devcontainers --- .devcontainer/.dockerignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .devcontainer/.dockerignore diff --git a/.devcontainer/.dockerignore b/.devcontainer/.dockerignore deleted file mode 100644 index 9bf06c0..0000000 --- a/.devcontainer/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -../Gemfile.lock -../vendor \ No newline at end of file From 929cb8ee6e0ec508454b5a2a193a2991e0e57ea2 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 08:15:50 +0000 Subject: [PATCH 18/20] add some empty lines --- .gitignore | 2 +- Gemfile | 2 +- lib/redis_mutex/standard_mutex.rb | 2 +- lib/redis_mutex/windowed_mutex.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index df45fab..e5f8948 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ pkg Gemfile.lock -vendor \ No newline at end of file +vendor diff --git a/Gemfile b/Gemfile index 8926307..d65e2a6 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,3 @@ source 'http://rubygems.org' -gemspec \ No newline at end of file +gemspec diff --git a/lib/redis_mutex/standard_mutex.rb b/lib/redis_mutex/standard_mutex.rb index 7da0e61..9757b5a 100644 --- a/lib/redis_mutex/standard_mutex.rb +++ b/lib/redis_mutex/standard_mutex.rb @@ -63,4 +63,4 @@ def standard_cleanup_set(_ = {}) nil end end -end \ No newline at end of file +end diff --git a/lib/redis_mutex/windowed_mutex.rb b/lib/redis_mutex/windowed_mutex.rb index a337706..dbf24a3 100644 --- a/lib/redis_mutex/windowed_mutex.rb +++ b/lib/redis_mutex/windowed_mutex.rb @@ -45,4 +45,4 @@ def windowed_cleanup_set(_ = {}) nil end end -end \ No newline at end of file +end From 9e1b338c62267c074bec0f8336e846befc6a8f72 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 08:16:12 +0000 Subject: [PATCH 19/20] I don't want a spec file to appear as a diff for this --- spec/redis_mutex_macro_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/redis_mutex_macro_spec.rb b/spec/redis_mutex_macro_spec.rb index 306ef06..09e1661 100644 --- a/spec/redis_mutex_macro_spec.rb +++ b/spec/redis_mutex_macro_spec.rb @@ -2,8 +2,8 @@ class C include RedisMutex::Macro - auto_mutex :run_singularly, :block => 0, :after_failure => lambda {|id| return "failure: #{id}" } + def run_singularly(id) sleep 0.1 return "success: #{id}" From c8b585b5629f953b58ec738f5d4ef2771c55ff25 Mon Sep 17 00:00:00 2001 From: James Woodrow Date: Tue, 25 Mar 2025 08:26:10 +0000 Subject: [PATCH 20/20] fix an error in ruby 2.1 --- .devcontainer/docker-compose.yml | 2 +- lib/redis_mutex/cumulative_mutex.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 05df563..80e0b9b 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -31,7 +31,7 @@ services: app: build: context: .. - dockerfile: .devcontainer/Dockerfile + dockerfile: .devcontainer/Dockerfile.2.1 image: redis-mutex container_name: redis-mutex pull_policy: build diff --git a/lib/redis_mutex/cumulative_mutex.rb b/lib/redis_mutex/cumulative_mutex.rb index d851c19..dabe9d2 100644 --- a/lib/redis_mutex/cumulative_mutex.rb +++ b/lib/redis_mutex/cumulative_mutex.rb @@ -32,7 +32,7 @@ def cumulative_try_lock end def cumulative_locked?(options = {}) - key_count(now: options[:now] || Time.now.to_i) >= options[:limit] || @limit + key_count(now: options[:now] || Time.now.to_i) >= (options[:limit] || @limit) end def cumulative_unlock(force = false)