diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4c8ff..a254b03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ## 3.4.0 - Add `retry_if` option to support custom retry predicates, including checks against wrapped `exception.cause` values. +- Add opt-in infinite retries via `tries: :infinite`, requiring a finite `max_elapsed_time` safety bound. ## 3.3.0 diff --git a/README.md b/README.md index cbbce5d..becf2a1 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Here are the available options, in some vague order of relevance to most common | Option | Default | Definition | | ---------------------- | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`tries`** | `3` | Number of attempts to make at running your code block (includes initial attempt). | +| **`tries`** | `3` | Number of attempts to make at running your code block (includes initial attempt). Pass `:infinite` to keep retrying until success or until `max_elapsed_time` is reached. | | **`on`** | `[StandardError]` | Type of exceptions to retry. [Read more](#configuring-which-options-to-retry-with-on). | | **`retry_if`** | `nil` | Callable (for example a `Proc` or lambda) that receives the rescued exception and returns true/false to decide whether to retry. [Read more](#advanced-retry-matching-with-retry_if). | | **`on_retry`** | `nil` | `Proc` to call after each try is rescued. [Read more](#callbacks). | @@ -233,6 +233,19 @@ end This example makes 5 total attempts. If the first attempt fails, the 2nd attempt occurs 0.5 seconds later. +### Infinite Retries (Opt-in) + +You can opt in to infinite retries with `tries: :infinite`. This is useful for long-running worker processes where retrying should continue until success, but it should be used carefully. + +```ruby +Retriable.retriable(tries: :infinite, max_elapsed_time: 300) do + # code here... +end +``` + +`max_elapsed_time` must be a finite number when using `tries: :infinite`. +Retriable raises `ArgumentError` if `max_elapsed_time` is unbounded. + ### Turn off Exponential Backoff Exponential backoff is enabled by default. If you want to simply retry code every second, 5 times maximum, you can do this: diff --git a/lib/retriable.rb b/lib/retriable.rb index f1ea6df..e742209 100644 --- a/lib/retriable.rb +++ b/lib/retriable.rb @@ -48,25 +48,21 @@ def retriable(opts = {}, &block) end local_config.validate! - - tries = local_config.tries - intervals = build_intervals(local_config, tries) timeout = local_config.timeout on = local_config.on retry_if = local_config.retry_if on_retry = local_config.on_retry sleep_disabled = local_config.sleep_disabled max_elapsed_time = local_config.max_elapsed_time + max_tries, interval_for = retry_plan(local_config) exception_list = on.is_a?(Hash) ? on.keys : on exception_list = [*exception_list] start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) elapsed_time = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time } - tries = intervals.size + 1 - execute_tries( - tries: tries, intervals: intervals, timeout: timeout, + max_tries: max_tries, interval_for: interval_for, timeout: timeout, exception_list: exception_list, on: on, retry_if: retry_if, on_retry: on_retry, elapsed_time: elapsed_time, max_elapsed_time: max_elapsed_time, sleep_disabled: sleep_disabled, &block @@ -74,22 +70,22 @@ def retriable(opts = {}, &block) end def execute_tries( # rubocop:disable Metrics/ParameterLists - tries:, intervals:, timeout:, exception_list:, + max_tries:, interval_for:, timeout:, exception_list:, on:, retry_if:, on_retry:, elapsed_time:, max_elapsed_time:, sleep_disabled:, &block ) - tries.times do |index| - try = index + 1 - + try = 0 + loop do + try += 1 begin return call_with_timeout(timeout, try, &block) rescue *exception_list => e raise unless retriable_exception?(e, on, exception_list, retry_if) - interval = intervals[index] + interval = interval_for.call(try - 1) call_on_retry(on_retry, e, try, elapsed_time.call, interval) elapsed_interval = sleep_disabled == true ? 0 : interval - raise unless can_retry?(try, tries, elapsed_time.call, elapsed_interval, max_elapsed_time) + raise unless can_retry?(try, max_tries, elapsed_time.call, elapsed_interval, max_elapsed_time) sleep interval if sleep_disabled != true end @@ -108,6 +104,38 @@ def build_intervals(local_config, tries) ).intervals end + def retry_plan(local_config) + return infinite_retry_plan(local_config) if local_config.tries == :infinite + + intervals = build_intervals(local_config, local_config.tries) + [intervals.size + 1, ->(i) { intervals[i] }] + end + + def infinite_retry_plan(local_config) + unless finite_number?(local_config.max_elapsed_time) + raise ArgumentError, + "max_elapsed_time must be finite when tries is :infinite" + end + + [nil, infinite_interval_provider(local_config)] + end + + def infinite_interval_provider(local_config) + if local_config.intervals + raise ArgumentError, "intervals must not be empty for infinite retries" if local_config.intervals.empty? + + custom = local_config.intervals + return ->(i) { custom.fetch(i) { custom.last } } + end + + ExponentialBackoff.new( + base_interval: local_config.base_interval, + multiplier: local_config.multiplier, + max_interval: local_config.max_interval, + rand_factor: local_config.rand_factor, + ).interval_provider + end + def call_with_timeout(timeout, try) return Timeout.timeout(timeout) { yield(try) } if timeout @@ -120,13 +148,17 @@ def call_on_retry(on_retry, exception, try, elapsed_time, interval) on_retry.call(exception, try, elapsed_time, interval) end - def can_retry?(try, tries, elapsed_time, interval, max_elapsed_time) - return false unless try < tries + def can_retry?(try, max_tries, elapsed_time, interval, max_elapsed_time) + return false if max_tries && try >= max_tries return true if max_elapsed_time.nil? (elapsed_time + interval) <= max_elapsed_time end + def finite_number?(value) + value.is_a?(Numeric) && (!value.respond_to?(:finite?) || value.finite?) + end + # When `on` is a Hash, we need to verify the exception matches a pattern. # For any non-Hash `on` value (e.g., Array of classes, single Exception class, # or Module), the `rescue *exception_list` clause already guarantees the @@ -207,9 +239,13 @@ def override_contexts :validate_context_override_options, :execute_tries, :build_intervals, + :retry_plan, + :infinite_retry_plan, + :infinite_interval_provider, :call_with_timeout, :call_on_retry, :can_retry?, + :finite_number?, :retriable_exception?, :hash_exception_match?, :apply_override_options, diff --git a/lib/retriable/config.rb b/lib/retriable/config.rb index 48fb514..031dc7e 100644 --- a/lib/retriable/config.rb +++ b/lib/retriable/config.rb @@ -54,8 +54,13 @@ def validate! validate_optional_non_negative_number(:timeout, timeout) validate_intervals return if intervals + return validate_backoff_options if tries == :infinite validate_positive_integer(:tries, tries) + validate_backoff_options + end + + def validate_backoff_options validate_non_negative_number(:base_interval, base_interval) validate_non_negative_number(:multiplier, multiplier) validate_non_negative_number(:max_interval, max_interval) diff --git a/lib/retriable/exponential_backoff.rb b/lib/retriable/exponential_backoff.rb index f9bd7d1..f936693 100644 --- a/lib/retriable/exponential_backoff.rb +++ b/lib/retriable/exponential_backoff.rb @@ -29,13 +29,25 @@ def initialize(opts = {}) end def intervals - intervals = Array.new(tries) do |iteration| - [base_interval * (multiplier**iteration), max_interval].min - end + Array.new(tries) { |iteration| interval_for(iteration) } + end + + def interval_for(iteration) + interval = [base_interval * (multiplier**iteration), max_interval].min + return interval if rand_factor.zero? + + randomize(interval) + end - return intervals if rand_factor.zero? + def interval_provider + raw_interval = base_interval - intervals.map { |i| randomize(i) } + lambda do |_iteration| + interval = [raw_interval, max_interval].min + raw_interval = next_raw_interval(raw_interval) + + rand_factor.zero? ? interval : randomize(interval) + end end private @@ -70,6 +82,12 @@ def finite_number?(value) value.is_a?(Numeric) && value.to_f.finite? end + def next_raw_interval(raw_interval) + return max_interval if multiplier >= 1 && raw_interval >= max_interval + + raw_interval * multiplier + end + def randomize(interval) delta = rand_factor * interval.to_f min = interval - delta diff --git a/spec/exponential_backoff_spec.rb b/spec/exponential_backoff_spec.rb index 6bf7a41..bc6801d 100644 --- a/spec/exponential_backoff_spec.rb +++ b/spec/exponential_backoff_spec.rb @@ -23,16 +23,16 @@ it "generates 10 randomized intervals" do expect(described_class.new(tries: 9).intervals).to eq([ - 0.5244067512211441, - 0.9113920238761231, - 1.2406087918999114, - 1.7632403621664823, - 2.338001204738311, - 4.350816718580626, - 5.339852157217869, - 11.889873261212443, - 18.756037881636484, - ]) + 0.5244067512211441, + 0.9113920238761231, + 1.2406087918999114, + 1.7632403621664823, + 2.338001204738311, + 4.350816718580626, + 5.339852157217869, + 11.889873261212443, + 18.756037881636484, + ]) end it "generates defined number of intervals" do @@ -41,18 +41,18 @@ it "generates intervals with a defined base interval" do expect(described_class.new(base_interval: 1).intervals).to eq([ - 1.0488135024422882, - 1.8227840477522461, - 2.4812175837998227, - ]) + 1.0488135024422882, + 1.8227840477522461, + 2.4812175837998227, + ]) end it "generates intervals with a defined multiplier" do expect(described_class.new(multiplier: 1).intervals).to eq([ - 0.5244067512211441, - 0.607594682584082, - 0.5513816852888495, - ]) + 0.5244067512211441, + 0.607594682584082, + 0.5513816852888495, + ]) end it "generates intervals with a defined max interval" do @@ -61,14 +61,21 @@ it "generates intervals with a defined rand_factor" do expect(described_class.new(rand_factor: 0.2).intervals).to eq([ - 0.5097627004884576, - 0.8145568095504492, - 1.1712435167599646, - ]) + 0.5097627004884576, + 0.8145568095504492, + 1.1712435167599646, + ]) end it "generates 10 non-randomized intervals" do non_random_intervals = 9.times.inject([0.5]) { |memo, _i| memo + [memo.last * 1.5] } expect(described_class.new(tries: 10, rand_factor: 0.0).intervals).to eq(non_random_intervals) end + + it "provides capped intervals lazily" do + interval_for = described_class.new(base_interval: 1.0, multiplier: 2.0, max_interval: 4.0, rand_factor: 0.0) + .interval_provider + + expect(Array.new(5) { |index| interval_for.call(index) }).to eq([1.0, 2.0, 4.0, 4.0, 4.0]) + end end diff --git a/spec/retriable_spec.rb b/spec/retriable_spec.rb index 979c8ff..743073c 100644 --- a/spec/retriable_spec.rb +++ b/spec/retriable_spec.rb @@ -84,6 +84,56 @@ def increment_tries_with_exception(exception_class = nil) expect(@tries).to eq(10) end + it "supports infinite retries until the block succeeds" do + described_class.retriable(tries: :infinite) do + increment_tries + raise StandardError if @tries < 5 + end + + expect(@tries).to eq(5) + end + + it "stops infinite retries at max_elapsed_time" do + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeline = [ + start_time, + start_time, + start_time, + start_time + 0.01, + start_time + 0.01, + start_time + 0.02, + start_time + 0.02, + ] + allow(Process).to receive(:clock_gettime).with(Process::CLOCK_MONOTONIC) { timeline.shift || timeline.last } + + expect do + described_class.retriable( + tries: :infinite, + base_interval: 0.01, + multiplier: 1.0, + rand_factor: 0.0, + sleep_disabled: true, + max_elapsed_time: 0.015, + ) do + increment_tries_with_exception + end + end.to raise_error(StandardError) + + expect(@tries).to eq(3) + end + + it "raises ArgumentError for infinite retries without a finite max_elapsed_time" do + expect do + described_class.retriable(tries: :infinite, max_elapsed_time: Float::INFINITY) { increment_tries } + end.to raise_error(ArgumentError, /max_elapsed_time/) + end + + it "raises ArgumentError for infinite retries with empty intervals" do + expect do + described_class.retriable(tries: :infinite, intervals: []) { increment_tries_with_exception } + end.to raise_error(ArgumentError, /intervals/) + end + it "will timeout after 1 second" do expect { described_class.retriable(timeout: 1) { sleep(1.1) } }.to raise_error(Timeout::Error) end @@ -195,6 +245,16 @@ def increment_tries_with_exception(exception_class = nil) expect(@next_interval_table[3]).to eq(0.3) expect(@next_interval_table[4]).to be_nil end + + it "allows non-integer tries when intervals are provided" do + expect do + described_class.retriable(intervals: [0.1], tries: :ignored) do + increment_tries_with_exception + end + end.to raise_error(StandardError) + + expect(@tries).to eq(2) + end end context "with an array :on parameter" do