diff --git a/Gemfile b/Gemfile index 6e637740a..3d1af98e6 100644 --- a/Gemfile +++ b/Gemfile @@ -62,7 +62,9 @@ gem "lograge" # For distributed tracing and telemetry gem "opentelemetry-exporter-otlp", "~> 0.34.0" +gem "opentelemetry-exporter-otlp-metrics", "~> 0.10.0" gem "opentelemetry-instrumentation-all", "~> 0.94.0" +gem "opentelemetry-metrics-sdk", "~> 0.15.0" gem "opentelemetry-propagator-xray", "~> 0.27.0" gem "opentelemetry-sdk", "~> 1.12" diff --git a/Gemfile.lock b/Gemfile.lock index 40bc45140..b32e88194 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -373,6 +373,15 @@ GEM opentelemetry-common (~> 0.20) opentelemetry-sdk (~> 1.10) opentelemetry-semantic_conventions + opentelemetry-exporter-otlp-metrics (0.10.0) + google-protobuf (>= 3.18, < 5.0) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-metrics-api (~> 0.2) + opentelemetry-metrics-sdk (~> 0.5) + opentelemetry-sdk (~> 1.2) + opentelemetry-semantic_conventions opentelemetry-helpers-mysql (0.6.0) opentelemetry-api (~> 1.7) opentelemetry-common (~> 0.21) @@ -527,6 +536,12 @@ GEM opentelemetry-helpers-sql-processor opentelemetry-instrumentation-base (~> 0.25) opentelemetry-semantic_conventions (>= 1.8.0) + opentelemetry-metrics-api (0.6.0) + opentelemetry-api (~> 1.0) + opentelemetry-metrics-sdk (0.15.0) + opentelemetry-api (~> 1.1) + opentelemetry-metrics-api (~> 0.2) + opentelemetry-sdk (~> 1.2) opentelemetry-propagator-xray (0.27.0) opentelemetry-api (~> 1.7) opentelemetry-registry (0.6.0) @@ -800,7 +815,9 @@ DEPENDENCIES omniauth-rails_csrf_protection omniauth_govuk_one_login! opentelemetry-exporter-otlp (~> 0.34.0) + opentelemetry-exporter-otlp-metrics (~> 0.10.0) opentelemetry-instrumentation-all (~> 0.94.0) + opentelemetry-metrics-sdk (~> 0.15.0) opentelemetry-propagator-xray (~> 0.27.0) opentelemetry-sdk (~> 1.12) pg (~> 1.6) @@ -947,6 +964,7 @@ CHECKSUMS opentelemetry-api (1.10.0) sha256=99ee7c829b18381c31a817ee9bf6a160d737542d99cb8da55d443336d266bfa9 opentelemetry-common (0.25.0) sha256=73915362e58d337fc92acbe1abfdaee1f725442527125fdb2af1420417f1149d opentelemetry-exporter-otlp (0.34.0) sha256=3b3cdf4329ba30f4389d849c7f13b8f9f983ecb4a030031c03997dffae1e2a60 + opentelemetry-exporter-otlp-metrics (0.10.0) sha256=d8cbff9b8a3391eb61486b8be9b6ad74e3b9306a3c60fb4c906b28bc857167c8 opentelemetry-helpers-mysql (0.6.0) sha256=7eeb5e6950c434775a8cf28b5fde4defc12e8b865c86479ce3119fcf593d9337 opentelemetry-helpers-sql (0.4.0) sha256=b10e8c3a2cca28a98af951bbb3e4efdc59e68b25ba0825e055574af543420afb opentelemetry-helpers-sql-processor (0.5.0) sha256=b199241bc9451fcbd9f00b2f454830af19d4ca27c2219ea379c9b0d53cd0e0f1 @@ -996,6 +1014,8 @@ CHECKSUMS opentelemetry-instrumentation-sidekiq (0.29.0) sha256=b1d2a0cb9041a5e14239fe7c94d99e3dd07f870e2759460ab63592d7cdd8aadc opentelemetry-instrumentation-sinatra (0.30.0) sha256=b67301153420f43264a0c68cdb3ca5bd77467cf5054e57b83a2bf891aaaa0361 opentelemetry-instrumentation-trilogy (0.69.0) sha256=0676dd720eeab284abfa52f273967442156fcac7084a1e1411373cf14ec026ad + opentelemetry-metrics-api (0.6.0) sha256=b9300821680a1370684098cb030c18423dd55909ea0206faadfa7bc47362df87 + opentelemetry-metrics-sdk (0.15.0) sha256=611a9cd9f473c461095c7401b8c25f9774160d286a1acbfcbf044da2972aeada opentelemetry-propagator-xray (0.27.0) sha256=753f756c7ad3146f182d428b06041084eecc77769edfd280f365e0bc09b9c4d1 opentelemetry-registry (0.6.0) sha256=5d3ed32ab9eee0fbdb30d4f0d0bb61ad11a4040b267b475ae815b80a8498a728 opentelemetry-sdk (1.12.0) sha256=a224abe0c59023d41cb7ac1c634d9d28843907efcd045ed1ae320796c48b864b diff --git a/app/services/form_submission_service.rb b/app/services/form_submission_service.rb index c2b15b6b1..2a3be19ce 100644 --- a/app/services/form_submission_service.rb +++ b/app/services/form_submission_service.rb @@ -118,6 +118,8 @@ def enqueue_deliver_submission_job(job_class) raise StandardError, "Failed to enqueue submission for reference #{submission_reference}#{message_suffix}" end + Metrics.record_submission(form_id: form.id, form_name: form.name, mode:) + submission end diff --git a/app/services/metrics.rb b/app/services/metrics.rb new file mode 100644 index 000000000..f6a0f2d6e --- /dev/null +++ b/app/services/metrics.rb @@ -0,0 +1,45 @@ +module Metrics + METER_NAME = "forms-runner".freeze + METER_VERSION = "1.0".freeze + + # Forms with these names are used for automated testing of the platform + TEST_FORM_NAME_PATTERNS = [/\Acapybara/, /smoke/, /\As3/].freeze + + class << self + def record_submission(form_id:, form_name:, mode:) + submission_counter.add( + 1, + attributes: { + "form.id" => form_id.to_s, + "form.submission.mode" => mode_label(form_name:, mode:), + }, + ) + rescue StandardError => e + Sentry.capture_exception(e) + end + + private + + def mode_label(form_name:, mode:) + return "test" if test_form?(form_name) + + mode.to_s + end + + def test_form?(form_name) + TEST_FORM_NAME_PATTERNS.any? { |pattern| pattern.match?(form_name.to_s) } + end + + def submission_counter + @submission_counter ||= meter.create_counter( + "form.submission.created", + unit: "{submission}", + description: "The number of form submissions queued for delivery", + ) + end + + def meter + @meter ||= OpenTelemetry.meter_provider.meter(METER_NAME, version: METER_VERSION) + end + end +end diff --git a/config/initializers/opentelemetry.rb b/config/initializers/opentelemetry.rb index eda8872b3..c68fb935c 100644 --- a/config/initializers/opentelemetry.rb +++ b/config/initializers/opentelemetry.rb @@ -1,5 +1,7 @@ require "opentelemetry/sdk" require "opentelemetry/instrumentation/all" +require "opentelemetry-metrics-sdk" +require "opentelemetry-exporter-otlp-metrics" return unless ENV["ENABLE_OTEL"] == "true" @@ -12,6 +14,14 @@ c.id_generator = OpenTelemetry::Propagator::XRay::IDGenerator end + unless ENV.fetch("OTEL_METRICS_EXPORTER", "otlp") == "none" + c.add_metric_reader( + OpenTelemetry::SDK::Metrics::Export::PeriodicMetricReader.new( + exporter: OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new, + ), + ) + end + # Disable logging for Rake tasks to avoid cluttering output c.logger = Logger.new(File::NULL) if Rails.const_defined?(:Rake) && Rake.application.top_level_tasks.any? end diff --git a/spec/services/form_submission_service_spec.rb b/spec/services/form_submission_service_spec.rb index 158bdb406..a795eedb2 100644 --- a/spec/services/form_submission_service_spec.rb +++ b/spec/services/form_submission_service_spec.rb @@ -102,6 +102,16 @@ expect(log_line["submission_reference"]).to eq(reference) end + it "records a submission count metric" do + expect(Metrics).to receive(:record_submission).with( + form_id: form.id, + form_name: form.name, + mode:, + ) + + service.submit + end + shared_examples "logging" do it "logs submission" do allow(LogEventService).to receive(:log_submit).once @@ -153,6 +163,12 @@ allow(SendS3SubmissionJob).to receive(:perform_later).and_yield(instance_double(SendS3SubmissionJob, successfully_enqueued?: false, enqueue_error:)) end + it "does not record a submission count metric" do + expect(Metrics).not_to receive(:record_submission) + + expect { service.submit }.to raise_error(StandardError) + end + context "and there is no enqueue error" do it "raises an error" do expect { service.submit }.to not_change(Submission, :count).and raise_error(StandardError, "Failed to enqueue submission for reference #{reference}") @@ -226,6 +242,12 @@ allow(SendSubmissionJob).to receive(:perform_later).and_yield(instance_double(SendSubmissionJob, successfully_enqueued?: false, enqueue_error:)) end + it "does not record a submission count metric" do + expect(Metrics).not_to receive(:record_submission) + + expect { service.submit }.to raise_error(StandardError) + end + context "and there is no enqueue error" do it "raises an error" do expect { service.submit }.to not_change(Submission, :count).and(not_change(Delivery, :count)).and raise_error(StandardError, "Failed to enqueue submission for reference #{reference}") diff --git a/spec/services/metrics_spec.rb b/spec/services/metrics_spec.rb new file mode 100644 index 000000000..cc010df1e --- /dev/null +++ b/spec/services/metrics_spec.rb @@ -0,0 +1,141 @@ +require "rails_helper" +require "opentelemetry-metrics-sdk" + +describe Metrics do + let(:meter_provider) { OpenTelemetry::SDK::Metrics::MeterProvider.new } + let(:metric_exporter) { OpenTelemetry::SDK::Metrics::Export::InMemoryMetricPullExporter.new } + let(:form_id) { 42 } + let(:form_name) { "Apply for a juggling licence" } + let(:mode) { Mode.new("form") } + + before do + allow(OpenTelemetry).to receive(:meter_provider).and_return(meter_provider) + meter_provider.add_metric_reader(metric_exporter) + reset_memoized_instruments + end + + after do + reset_memoized_instruments + end + + describe ".record_submission" do + it "records a submission count metric" do + described_class.record_submission(form_id:, form_name:, mode:) + + expect(exported_data_points).to contain_exactly( + have_attributes( + value: 1, + attributes: { + "form.id" => form_id.to_s, + "form.submission.mode" => "form", + }, + ), + ) + end + + context "when mode is preview" do + let(:mode) { Mode.new("preview-live") } + + it "records a metric with the preview mode label" do + described_class.record_submission(form_id:, form_name:, mode:) + + expect(exported_data_points).to contain_exactly( + have_attributes( + value: 1, + attributes: include("form.submission.mode" => "preview-live"), + ), + ) + end + end + + it "accumulates counts for the same form" do + 2.times { described_class.record_submission(form_id:, form_name:, mode:) } + + expect(exported_data_points).to contain_exactly( + have_attributes( + value: 2, + attributes: include("form.id" => form_id.to_s), + ), + ) + end + + it "records separate counts per form" do + described_class.record_submission(form_id:, form_name:, mode:) + described_class.record_submission(form_id: 99, form_name:, mode:) + + expect(exported_data_points).to contain_exactly( + have_attributes( + value: 1, + attributes: include("form.id" => form_id.to_s), + ), + have_attributes( + value: 1, + attributes: include("form.id" => "99"), + ), + ) + end + + it "records separate counts per mode" do + described_class.record_submission(form_id:, form_name:, mode:) + described_class.record_submission(form_id:, form_name:, mode: Mode.new("preview-draft")) + + expect(exported_data_points).to contain_exactly( + have_attributes( + value: 1, + attributes: include("form.submission.mode" => "form"), + ), + have_attributes( + value: 1, + attributes: include("form.submission.mode" => "preview-draft"), + ), + ) + end + + context "when the form is a test form" do + [ + "capybara test form", + "Automated smoke test form", + "s3 submission test form", + ].each do |test_form_name| + context "with the name #{test_form_name.inspect}" do + let(:form_name) { test_form_name } + + it "records a metric with the test mode label" do + described_class.record_submission(form_id:, form_name:, mode:) + + expect(exported_data_points).to contain_exactly( + have_attributes( + value: 1, + attributes: include("form.submission.mode" => "test"), + ), + ) + end + end + end + end + + context "when recording the metric raises an error" do + let(:error) { StandardError.new("metrics unavailable") } + + before do + allow(described_class).to receive(:submission_counter).and_raise(error) + end + + it "captures the exception in Sentry and does not raise" do + expect(Sentry).to receive(:capture_exception).with(error) + + expect { described_class.record_submission(form_id:, form_name:, mode:) }.not_to raise_error + end + end + end + + def exported_data_points + metric_exporter.pull + metric_exporter.metric_snapshots.flat_map(&:data_points) + end + + def reset_memoized_instruments + described_class.instance_variable_set(:@meter, nil) + described_class.instance_variable_set(:@submission_counter, nil) + end +end