Improve scoop

.rubocop.yml

@@ -105,6 +105,7 @@ Metrics/ClassLength:
     - 'lib/cypress/demographics_randomizer.rb'
     - 'lib/cypress/expected_results_calculator.rb'
     - 'lib/cypress/population_clone_job.rb'
+    - 'lib/cypress/scoop_and_filter.rb'
     - 'lib/validators/cat3_population_validator.rb'
 Metrics/CyclomaticComplexity:
   Max: 12

Gemfile

@@ -5,7 +5,7 @@ source 'https://rubygems.org'
 ruby '3.4.9'
 
 # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
-gem 'rails', '~> 8.1.0'
+gem 'rails', '~> 8.0.0', '>= 8.0.2.1'
 
 gem 'propshaft'
 
@@ -126,7 +126,6 @@ group :development, :test do
   # remove scss_lint, incompatible with sass dependency upgrades
   # gem 'scss_lint', require: false
   gem 'selenium-webdriver'
-  gem 'webrick'
 end
 
 group :development do

Gemfile.lock

@@ -1,75 +1,72 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    action_text-trix (2.1.19)
-      railties
-    actioncable (8.1.3)
-      actionpack (= 8.1.3)
-      activesupport (= 8.1.3)
+    actioncable (8.0.5)
+      actionpack (= 8.0.5)
+      activesupport (= 8.0.5)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
       zeitwerk (~> 2.6)
-    actionmailbox (8.1.3)
-      actionpack (= 8.1.3)
-      activejob (= 8.1.3)
-      activerecord (= 8.1.3)
-      activestorage (= 8.1.3)
-      activesupport (= 8.1.3)
+    actionmailbox (8.0.5)
+      actionpack (= 8.0.5)
+      activejob (= 8.0.5)
+      activerecord (= 8.0.5)
+      activestorage (= 8.0.5)
+      activesupport (= 8.0.5)
       mail (>= 2.8.0)
-    actionmailer (8.1.3)
-      actionpack (= 8.1.3)
-      actionview (= 8.1.3)
-      activejob (= 8.1.3)
-      activesupport (= 8.1.3)
+    actionmailer (8.0.5)
+      actionpack (= 8.0.5)
+      actionview (= 8.0.5)
+      activejob (= 8.0.5)
+      activesupport (= 8.0.5)
       mail (>= 2.8.0)
       rails-dom-testing (~> 2.2)
-    actionpack (8.1.3)
-      actionview (= 8.1.3)
-      activesupport (= 8.1.3)
+    actionpack (8.0.5)
+      actionview (= 8.0.5)
+      activesupport (= 8.0.5)
       nokogiri (>= 1.8.5)
       rack (>= 2.2.4)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
       useragent (~> 0.16)
-    actiontext (8.1.3)
-      action_text-trix (~> 2.1.15)
-      actionpack (= 8.1.3)
-      activerecord (= 8.1.3)
-      activestorage (= 8.1.3)
-      activesupport (= 8.1.3)
+    actiontext (8.0.5)
+      actionpack (= 8.0.5)
+      activerecord (= 8.0.5)
+      activestorage (= 8.0.5)
+      activesupport (= 8.0.5)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (8.1.3)
-      activesupport (= 8.1.3)
+    actionview (8.0.5)
+      activesupport (= 8.0.5)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    activejob (8.1.3)
-      activesupport (= 8.1.3)
+    activejob (8.0.5)
+      activesupport (= 8.0.5)
       globalid (>= 0.3.6)
-    activemodel (8.1.3)
-      activesupport (= 8.1.3)
-    activerecord (8.1.3)
-      activemodel (= 8.1.3)
-      activesupport (= 8.1.3)
+    activemodel (8.0.5)
+      activesupport (= 8.0.5)
+    activerecord (8.0.5)
+      activemodel (= 8.0.5)
+      activesupport (= 8.0.5)
       timeout (>= 0.4.0)
-    activestorage (8.1.3)
-      actionpack (= 8.1.3)
-      activejob (= 8.1.3)
-      activerecord (= 8.1.3)
-      activesupport (= 8.1.3)
+    activestorage (8.0.5)
+      actionpack (= 8.0.5)
+      activejob (= 8.0.5)
+      activerecord (= 8.0.5)
+      activesupport (= 8.0.5)
       marcel (~> 1.0)
-    activesupport (8.1.3)
+    activesupport (8.0.5)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
-      json
       logger (>= 1.4.2)
       minitest (>= 5.1)
       securerandom (>= 0.3)
@@ -265,7 +262,6 @@ GEM
       railties (>= 6.1.0)
     faker (3.8.0)
       i18n (>= 1.8.11, < 2)
-    ffi (1.17.4)
     ffi (1.17.4-aarch64-linux-gnu)
     ffi (1.17.4-aarch64-linux-musl)
     ffi (1.17.4-arm-linux-gnu)
@@ -333,7 +329,7 @@ GEM
       thor (>= 0.14, < 2.0)
     jquery-ui-rails (8.0.0)
       railties (>= 3.2.16)
-    json (2.19.4)
+    json (2.20.0)
     kaminari-core (1.2.2)
     kaminari-mongoid (1.0.2)
       kaminari-core (~> 1.0)
@@ -373,11 +369,10 @@ GEM
     mini_magick (5.3.1)
       logger
     mini_mime (1.1.5)
-    mini_portile2 (2.8.9)
     minitest (5.27.0)
-    minitest-rails (8.1.0)
+    minitest-rails (8.0.0)
       minitest (~> 5.20)
-      railties (>= 8.1.0, < 8.2.0)
+      railties (>= 8.0.0, < 8.1.0)
     minitest-reporters (1.8.0)
       ansi
       builder
@@ -415,9 +410,6 @@ GEM
     newrelic_rpm (10.4.0)
       logger
     nio4r (2.7.5)
-    nokogiri (1.19.4)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
     nokogiri (1.19.4-aarch64-linux-gnu)
       racc (~> 1.4)
     nokogiri (1.19.4-aarch64-linux-musl)
@@ -477,20 +469,20 @@ GEM
       rack (>= 1.3)
     rackup (2.3.1)
       rack (>= 3)
-    rails (8.1.3)
-      actioncable (= 8.1.3)
-      actionmailbox (= 8.1.3)
-      actionmailer (= 8.1.3)
-      actionpack (= 8.1.3)
-      actiontext (= 8.1.3)
-      actionview (= 8.1.3)
-      activejob (= 8.1.3)
-      activemodel (= 8.1.3)
-      activerecord (= 8.1.3)
-      activestorage (= 8.1.3)
-      activesupport (= 8.1.3)
+    rails (8.0.5)
+      actioncable (= 8.0.5)
+      actionmailbox (= 8.0.5)
+      actionmailer (= 8.0.5)
+      actionpack (= 8.0.5)
+      actiontext (= 8.0.5)
+      actionview (= 8.0.5)
+      activejob (= 8.0.5)
+      activemodel (= 8.0.5)
+      activerecord (= 8.0.5)
+      activestorage (= 8.0.5)
+      activesupport (= 8.0.5)
       bundler (>= 1.15.0)
-      railties (= 8.1.3)
+      railties (= 8.0.5)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -512,9 +504,9 @@ GEM
       ostruct
       require_all (~> 3.0)
       ruby-progressbar
-    railties (8.1.3)
-      actionpack (= 8.1.3)
-      activesupport (= 8.1.3)
+    railties (8.0.5)
+      actionpack (= 8.0.5)
+      activesupport (= 8.0.5)
       irb (~> 1.13)
       rackup (>= 1.0.0)
       rake (>= 12.2)
@@ -578,9 +570,6 @@ GEM
       logger
     ruby2_keywords (0.0.5)
     rubyzip (1.3.0)
-    sass-embedded (1.99.0)
-      google-protobuf (~> 4.31)
-      rake (>= 13)
     sass-embedded (1.99.0-aarch64-linux-gnu)
       google-protobuf (~> 4.31)
     sass-embedded (1.99.0-aarch64-linux-musl)
@@ -669,7 +658,6 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.9.2)
     websocket (1.2.11)
     websocket-driver (0.8.2)
       base64
@@ -689,7 +677,6 @@ PLATFORMS
   arm-linux-musl
   arm-linux-musleabihf
   arm64-darwin
-  ruby
   x86_64-darwin
   x86_64-linux-gnu
   x86_64-linux-musl
@@ -747,7 +734,7 @@ DEPENDENCIES
   pry
   pry-nav
   puma (~> 7.2.1)
-  rails (~> 8.1.0)
+  rails (~> 8.0.0, >= 8.0.2.1)
   rails-controller-testing
   rails-perftest
   rails_best_practices
@@ -771,7 +758,6 @@ DEPENDENCIES
   vmstat
   web-console (~> 4.2.0)
   webmock
-  webrick
 
 RUBY VERSION
    ruby 3.4.9p82

app/models/product_test.rb

@@ -92,11 +92,13 @@ def generate_patients(job_id = nil)
     if product.randomize_patients
       # If we're using a "slim test deck", don't pass in any random IDs
       random_ids = slim_test_deck? ? [] : gather_patient_ids
-      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'randomization_ids' => random_ids,
+      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids,
+                                      'scoop_and_filter' => true, 'randomization_ids' => random_ids,
                                       'include_virtual' => (measure_ids & APP_CONSTANTS['telehealth_ineligible_measures']),
                                       'randomize_demographics' => true, 'generate_provider' => product.c4_test, 'job_id' => job_id).perform
     else
-      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'disable_randomization' => true).perform
+      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids,
+                                      'scoop_and_filter' => true, 'disable_randomization' => true).perform
     end
   end
 

features/support/env.rb

@@ -86,7 +86,7 @@ def default_drivers
 
 default_drivers
 
-Capybara.server = :webrick
+Capybara.server = :puma, { Silent: true }
 Capybara.default_max_wait_time = 15
 # Capybara.ignore_hidden_elements = false
 

lib/cypress/patient_zipper.rb

@@ -79,7 +79,7 @@ def self.zip(file, patients, format)
           sf_patient.id = patient.id
           # CMS529 requires that entries point to the ecounters that they are related to
           sf_patient.add_encounter_ids_to_events if measures.map(&:hqmf_id).intersect?(APP_CONSTANTS['result_measures'].map(&:hqmf_id))
-          patient_scoop_and_filter.scoop_and_filter(sf_patient)
+          patient_scoop_and_filter.replace_negated_codes_with_valueset(sf_patient)
           z.put_next_entry("#{next_entry_path(patient, i)}.#{FORMAT_EXTENSIONS[format.to_sym]}")
           z << formatter.export(sf_patient)
         end

lib/cypress/population_clone_job.rb

@@ -47,6 +47,8 @@ def perform
         end
       end
 
+      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures) if @options['scoop_and_filter']
+
       # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different
       # providers for each patient
       provider = @test.provider if @test.instance_of?(MeasureTest)
@@ -96,6 +98,7 @@ def clone_and_save_patient(patient, prng, provider = nil, allow_dups: false)
         original_patient_id: patient.id,
         correlation_id: options['test_id']
       }
+      @patient_scoop_and_filter&.scoop_and_filter(cloned_patient, replace_negations: false, persist_scoop: true)
       unnumerify cloned_patient if patient.givenNames.map { |n| n =~ /\d/ }.any? || patient.familyName =~ /\d/
       cloned_patient.medical_record_number = next_medical_record_number unless options['disable_randomization']
       @test.reload