From 4715dcf7ee09495c6e4ad6529859e5d163d3be70 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Tue, 23 Jun 2026 11:57:39 -0400
Subject: [PATCH 01/14] restrict coded attributes to codes in measure too

---
 lib/cypress/population_clone_job.rb |  3 +++
 lib/cypress/scoop_and_filter.rb     | 24 +++++++++++++++++++++---
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/lib/cypress/population_clone_job.rb b/lib/cypress/population_clone_job.rb
index a6b5d2db7..6dced9ae2 100644
--- a/lib/cypress/population_clone_job.rb
+++ b/lib/cypress/population_clone_job.rb
@@ -47,6 +47,8 @@ def perform
         end
       end
 
+      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures, true)
+
       # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different
       # providers for each patient
       provider = @test.provider if @test.instance_of?(MeasureTest)
@@ -96,6 +98,7 @@ def clone_and_save_patient(patient, prng, provider = nil, allow_dups: false)
         original_patient_id: patient.id,
         correlation_id: options['test_id']
       }
+      @patient_scoop_and_filter.scoop_and_filter(cloned_patient)
       unnumerify cloned_patient if patient.givenNames.map { |n| n =~ /\d/ }.any? || patient.familyName =~ /\d/
       cloned_patient.medical_record_number = next_medical_record_number unless options['disable_randomization']
       @test.reload
diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 580fd4937..88fa68bb2 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -2,7 +2,8 @@
 
 module Cypress
   class ScoopAndFilter
-    def initialize(measures)
+    def initialize(measures, for_calculation = false)
+      @for_calculation = for_calculation
       @valuesets = measures.collect(&:value_sets).flatten.uniq
       @relevant_codes = codes_in_measures
       @de_category_statuses_for_measures = get_non_demographic_category_statuses(measures)
@@ -50,12 +51,29 @@ def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements
       # Return if all codes have been removed
       return if data_element.dataElementCodes.blank?
 
+      data_element.fields.keys.each do |field_name|
+        next if data_element[field_name] == nil
+
+        if field_name == 'diagnoses'
+          data_element.diagnoses.keep_if do |diagnosis|
+            @relevant_codes.include?(code: diagnosis.code.code, system: diagnosis.code.system)
+          end
+        elsif field_name == 'facilityLocations'
+          data_element.facilityLocations.keep_if do |facility_location|
+            @relevant_codes.include?(code: facility_location.code.code, system: facility_location.code.system)
+          end
+        end
+        next unless data_element.fields[field_name].type == QDM::Code
+
+        data_element[field_name] = nil unless @relevant_codes.include?(code: data_element[field_name].code, system: data_element[field_name].system)
+      end
+
       remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
       # Return if all codes and valuesets have been removed
       return if data_element.dataElementCodes.blank?
       return unless data_element.respond_to?('negationRationale') && data_element.negationRationale
 
-      replace_negated_code_with_valueset(data_element, multi_vs_negation_elements)
+      replace_negated_code_with_valueset(data_element, multi_vs_negation_elements) unless @for_calculation
       return if data_element.dataElementCodes.blank?
 
       # add data element valueset and other potentially relevant valueset descriptions
@@ -88,7 +106,7 @@ def remove_irrelevant_valuesets_and_add_description_to_data_element(data_element
       end
 
       vs = vsets.first
-      de.description = vs.display_name
+      de.description = vs.display_name unless @for_calculation
     end
 
     def value_set_appropriate_for_data_element(data_element, valueset_oid)

From 81f12f07daf269c68bf6ea48c296010ea2002703 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Thu, 25 Jun 2026 14:16:01 -0400
Subject: [PATCH 02/14] remove if blank

---
 lib/cypress/scoop_and_filter.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 88fa68bb2..b9a081ed0 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -58,10 +58,12 @@ def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements
           data_element.diagnoses.keep_if do |diagnosis|
             @relevant_codes.include?(code: diagnosis.code.code, system: diagnosis.code.system)
           end
+          data_element.diagnoses = nil if data_element.diagnoses.blank?
         elsif field_name == 'facilityLocations'
           data_element.facilityLocations.keep_if do |facility_location|
             @relevant_codes.include?(code: facility_location.code.code, system: facility_location.code.system)
           end
+          data_element.facilityLocations = nil if data_element.facilityLocations.blank?
         end
         next unless data_element.fields[field_name].type == QDM::Code
 

From f9cfb49de2c51ecb845863f093f162b7f655427a Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Wed, 8 Jul 2026 13:48:29 -0400
Subject: [PATCH 03/14] some places no longer need scoop and filter

---
 lib/cypress/create_download_zip.rb  | 2 --
 lib/cypress/patient_zipper.rb       | 2 --
 lib/cypress/population_clone_job.rb | 2 +-
 lib/cypress/scoop_and_filter.rb     | 2 +-
 4 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/lib/cypress/create_download_zip.rb b/lib/cypress/create_download_zip.rb
index f81050362..a1b413724 100644
--- a/lib/cypress/create_download_zip.rb
+++ b/lib/cypress/create_download_zip.rb
@@ -101,8 +101,6 @@ def self.create_c1_patient_zip(checklist_test)
           # TODO: R2P: format patients for export
           sf_patient = patient.clone
           sf_patient.id = patient.id
-          sf_measures = Measure.where(cms_id: measure_id)
-          Cypress::ScoopAndFilter.new(sf_measures).scoop_and_filter(sf_patient)
           add_file_to_zip(z, "sample patient for #{measure_id}.html", formatter.export(sf_patient))
         end
       end
diff --git a/lib/cypress/patient_zipper.rb b/lib/cypress/patient_zipper.rb
index 54a262ac4..945cc4891 100644
--- a/lib/cypress/patient_zipper.rb
+++ b/lib/cypress/patient_zipper.rb
@@ -64,7 +64,6 @@ class PatientZipper
     def self.zip(file, patients, format)
       patients = apply_sort_to patients
       measures, sd, ed = measure_start_end(patients)
-      patient_scoop_and_filter = Cypress::ScoopAndFilter.new(measures)
 
       # TODO: R2P: make sure patient exporter works (use correct one)
       formatter = if format.to_sym == :qrda
@@ -79,7 +78,6 @@ def self.zip(file, patients, format)
           sf_patient.id = patient.id
           # CMS529 requires that entries point to the ecounters that they are related to
           sf_patient.add_encounter_ids_to_events if measures.map(&:hqmf_id).intersect?(APP_CONSTANTS['result_measures'].map(&:hqmf_id))
-          patient_scoop_and_filter.scoop_and_filter(sf_patient)
           z.put_next_entry("#{next_entry_path(patient, i)}.#{FORMAT_EXTENSIONS[format.to_sym]}")
           z << formatter.export(sf_patient)
         end
diff --git a/lib/cypress/population_clone_job.rb b/lib/cypress/population_clone_job.rb
index 6dced9ae2..36b8c58bf 100644
--- a/lib/cypress/population_clone_job.rb
+++ b/lib/cypress/population_clone_job.rb
@@ -47,7 +47,7 @@ def perform
         end
       end
 
-      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures, true)
+      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures, for_calculation: true)
 
       # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different
       # providers for each patient
diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index b9a081ed0..913776a0e 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -2,7 +2,7 @@
 
 module Cypress
   class ScoopAndFilter
-    def initialize(measures, for_calculation = false)
+    def initialize(measures, for_calculation: false)
       @for_calculation = for_calculation
       @valuesets = measures.collect(&:value_sets).flatten.uniq
       @relevant_codes = codes_in_measures

From 76d613890070b5f512723dab31367785f20fa24b Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Wed, 8 Jul 2026 14:04:12 -0400
Subject: [PATCH 04/14] split into two methods

---
 lib/cypress/scoop_and_filter.rb | 43 +++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 913776a0e..3fc58efb1 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -46,41 +46,50 @@ def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements
       # keep if data_element code and codesystem is in one of the relevant_codes
       # Also keep all negated valuesets, we'll deal with those later
       data_element.dataElementCodes.keep_if do |de_code|
-        @relevant_codes.include?(code: de_code.code, system: de_code.system) || de_code.system == '1.2.3.4.5.6.7.8.9.10'
+        relevant_code?(de_code) || de_code.system == '1.2.3.4.5.6.7.8.9.10'
       end
       # Return if all codes have been removed
       return if data_element.dataElementCodes.blank?
 
+      scoop_and_filter_data_element_fields(data_element)
+      remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
+      # Return if all codes and valuesets have been removed
+      return if data_element.dataElementCodes.blank?
+      return unless data_element.respond_to?('negationRationale') && data_element.negationRationale
+
+      replace_negated_code_with_valueset(data_element, multi_vs_negation_elements) unless @for_calculation
+      return if data_element.dataElementCodes.blank?
+
+      # add data element valueset and other potentially relevant valueset descriptions
+      codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
+      Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
+    end
+
+    def scoop_and_filter_data_element_fields(data_element)
+      # Iterate through each field to see coded fields include relevant codes
       data_element.fields.keys.each do |field_name|
-        next if data_element[field_name] == nil
+        next if data_element[field_name].nil?
 
+        # Dianoses and Facility Locations are unique because they are arrays
         if field_name == 'diagnoses'
           data_element.diagnoses.keep_if do |diagnosis|
-            @relevant_codes.include?(code: diagnosis.code.code, system: diagnosis.code.system)
+            relevant_code?(diagnosis.code)
           end
           data_element.diagnoses = nil if data_element.diagnoses.blank?
         elsif field_name == 'facilityLocations'
           data_element.facilityLocations.keep_if do |facility_location|
-            @relevant_codes.include?(code: facility_location.code.code, system: facility_location.code.system)
+            relevant_code?(facility_location.code)
           end
           data_element.facilityLocations = nil if data_element.facilityLocations.blank?
         end
         next unless data_element.fields[field_name].type == QDM::Code
 
-        data_element[field_name] = nil unless @relevant_codes.include?(code: data_element[field_name].code, system: data_element[field_name].system)
+        data_element[field_name] = nil unless relevant_code?(data_element[field_name])
       end
+    end
 
-      remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
-      # Return if all codes and valuesets have been removed
-      return if data_element.dataElementCodes.blank?
-      return unless data_element.respond_to?('negationRationale') && data_element.negationRationale
-
-      replace_negated_code_with_valueset(data_element, multi_vs_negation_elements) unless @for_calculation
-      return if data_element.dataElementCodes.blank?
-
-      # add data element valueset and other potentially relevant valueset descriptions
-      codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
-      Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
+    def relevant_code?(code)
+      @relevant_codes.include?(code: code.code, system: code.system)
     end
 
     def data_element_category_and_status(data_element)
@@ -94,6 +103,7 @@ def data_element_used_by_measure(data_element)
       end.nil?
     end
 
+    # rubocop:disable Metrics/AbcSize
     def remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
       de = data_element
       vsets = if de.codes.any? { |code| code.system == '1.2.3.4.5.6.7.8.9.10' }
@@ -110,6 +120,7 @@ def remove_irrelevant_valuesets_and_add_description_to_data_element(data_element
       vs = vsets.first
       de.description = vs.display_name unless @for_calculation
     end
+    # rubocop:enable Metrics/AbcSize
 
     def value_set_appropriate_for_data_element(data_element, valueset_oid)
       @de_category_statuses_for_measures.include?(category: data_element['qdmCategory'], status: data_element['qdmStatus'], oid: valueset_oid)

From 9592083652552ad8d2ab24793b1784a5a33ab3c1 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Wed, 8 Jul 2026 14:06:34 -0400
Subject: [PATCH 05/14] update json gem

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 570f6d443..14443fc73 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -333,7 +333,7 @@ GEM
       thor (>= 0.14, < 2.0)
     jquery-ui-rails (8.0.0)
       railties (>= 3.2.16)
-    json (2.19.4)
+    json (2.20.0)
     kaminari-core (1.2.2)
     kaminari-mongoid (1.0.2)
       kaminari-core (~> 1.0)

From f83de968f430c13765d3eab2cf2aab3f4e58c466 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Wed, 8 Jul 2026 14:43:07 -0400
Subject: [PATCH 06/14] we do still need the scoop and filter before export (to
 account for negations)

---
 app/models/product_test.rb          | 6 ++++--
 lib/cypress/create_download_zip.rb  | 2 ++
 lib/cypress/patient_zipper.rb       | 2 ++
 lib/cypress/population_clone_job.rb | 4 ++--
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/app/models/product_test.rb b/app/models/product_test.rb
index 0c9c2b426..f9f046dda 100644
--- a/app/models/product_test.rb
+++ b/app/models/product_test.rb
@@ -92,11 +92,13 @@ def generate_patients(job_id = nil)
     if product.randomize_patients
       # If we're using a "slim test deck", don't pass in any random IDs
       random_ids = slim_test_deck? ? [] : gather_patient_ids
-      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'randomization_ids' => random_ids,
+      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids,
+                                      'scoop_and_filter' => true, 'randomization_ids' => random_ids,
                                       'include_virtual' => (measure_ids & APP_CONSTANTS['telehealth_ineligible_measures']),
                                       'randomize_demographics' => true, 'generate_provider' => product.c4_test, 'job_id' => job_id).perform
     else
-      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'disable_randomization' => true).perform
+      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids,
+                                      'scoop_and_filter' => true, 'disable_randomization' => true).perform
     end
   end
 
diff --git a/lib/cypress/create_download_zip.rb b/lib/cypress/create_download_zip.rb
index a1b413724..f81050362 100644
--- a/lib/cypress/create_download_zip.rb
+++ b/lib/cypress/create_download_zip.rb
@@ -101,6 +101,8 @@ def self.create_c1_patient_zip(checklist_test)
           # TODO: R2P: format patients for export
           sf_patient = patient.clone
           sf_patient.id = patient.id
+          sf_measures = Measure.where(cms_id: measure_id)
+          Cypress::ScoopAndFilter.new(sf_measures).scoop_and_filter(sf_patient)
           add_file_to_zip(z, "sample patient for #{measure_id}.html", formatter.export(sf_patient))
         end
       end
diff --git a/lib/cypress/patient_zipper.rb b/lib/cypress/patient_zipper.rb
index 945cc4891..54a262ac4 100644
--- a/lib/cypress/patient_zipper.rb
+++ b/lib/cypress/patient_zipper.rb
@@ -64,6 +64,7 @@ class PatientZipper
     def self.zip(file, patients, format)
       patients = apply_sort_to patients
       measures, sd, ed = measure_start_end(patients)
+      patient_scoop_and_filter = Cypress::ScoopAndFilter.new(measures)
 
       # TODO: R2P: make sure patient exporter works (use correct one)
       formatter = if format.to_sym == :qrda
@@ -78,6 +79,7 @@ def self.zip(file, patients, format)
           sf_patient.id = patient.id
           # CMS529 requires that entries point to the ecounters that they are related to
           sf_patient.add_encounter_ids_to_events if measures.map(&:hqmf_id).intersect?(APP_CONSTANTS['result_measures'].map(&:hqmf_id))
+          patient_scoop_and_filter.scoop_and_filter(sf_patient)
           z.put_next_entry("#{next_entry_path(patient, i)}.#{FORMAT_EXTENSIONS[format.to_sym]}")
           z << formatter.export(sf_patient)
         end
diff --git a/lib/cypress/population_clone_job.rb b/lib/cypress/population_clone_job.rb
index 36b8c58bf..180e4195a 100644
--- a/lib/cypress/population_clone_job.rb
+++ b/lib/cypress/population_clone_job.rb
@@ -47,7 +47,7 @@ def perform
         end
       end
 
-      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures, for_calculation: true)
+      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures, for_calculation: true) if @options['scoop_and_filter']
 
       # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different
       # providers for each patient
@@ -98,7 +98,7 @@ def clone_and_save_patient(patient, prng, provider = nil, allow_dups: false)
         original_patient_id: patient.id,
         correlation_id: options['test_id']
       }
-      @patient_scoop_and_filter.scoop_and_filter(cloned_patient)
+      @patient_scoop_and_filter&.scoop_and_filter(cloned_patient)
       unnumerify cloned_patient if patient.givenNames.map { |n| n =~ /\d/ }.any? || patient.familyName =~ /\d/
       cloned_patient.medical_record_number = next_medical_record_number unless options['disable_randomization']
       @test.reload

From 5df20a061d2b0f4d56e502daa2b1f403e9370fd3 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Wed, 8 Jul 2026 15:47:41 -0400
Subject: [PATCH 07/14] webrick not needed

---
 Gemfile      | 1 -
 Gemfile.lock | 2 --
 2 files changed, 3 deletions(-)

diff --git a/Gemfile b/Gemfile
index c5ffd6b65..a65e8cd79 100644
--- a/Gemfile
+++ b/Gemfile
@@ -126,7 +126,6 @@ group :development, :test do
   # remove scss_lint, incompatible with sass dependency upgrades
   # gem 'scss_lint', require: false
   gem 'selenium-webdriver'
-  gem 'webrick'
 end
 
 group :development do
diff --git a/Gemfile.lock b/Gemfile.lock
index 14443fc73..1c019b7a4 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -669,7 +669,6 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.9.2)
     websocket (1.2.11)
     websocket-driver (0.8.2)
       base64
@@ -771,7 +770,6 @@ DEPENDENCIES
   vmstat
   web-console (~> 4.2.0)
   webmock
-  webrick
 
 RUBY VERSION
    ruby 3.4.9p82

From 64ea4945c85abcff9fb988276984d5500a53386d Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Wed, 8 Jul 2026 15:59:12 -0400
Subject: [PATCH 08/14] use puma instead

---
 Gemfile.lock            | 9 ---------
 features/support/env.rb | 2 +-
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 1c019b7a4..2a8a96d3d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -265,7 +265,6 @@ GEM
       railties (>= 6.1.0)
     faker (3.8.0)
       i18n (>= 1.8.11, < 2)
-    ffi (1.17.4)
     ffi (1.17.4-aarch64-linux-gnu)
     ffi (1.17.4-aarch64-linux-musl)
     ffi (1.17.4-arm-linux-gnu)
@@ -373,7 +372,6 @@ GEM
     mini_magick (5.3.1)
       logger
     mini_mime (1.1.5)
-    mini_portile2 (2.8.9)
     minitest (5.27.0)
     minitest-rails (8.1.0)
       minitest (~> 5.20)
@@ -415,9 +413,6 @@ GEM
     newrelic_rpm (10.4.0)
       logger
     nio4r (2.7.5)
-    nokogiri (1.19.4)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
     nokogiri (1.19.4-aarch64-linux-gnu)
       racc (~> 1.4)
     nokogiri (1.19.4-aarch64-linux-musl)
@@ -578,9 +573,6 @@ GEM
       logger
     ruby2_keywords (0.0.5)
     rubyzip (1.3.0)
-    sass-embedded (1.99.0)
-      google-protobuf (~> 4.31)
-      rake (>= 13)
     sass-embedded (1.99.0-aarch64-linux-gnu)
       google-protobuf (~> 4.31)
     sass-embedded (1.99.0-aarch64-linux-musl)
@@ -688,7 +680,6 @@ PLATFORMS
   arm-linux-musl
   arm-linux-musleabihf
   arm64-darwin
-  ruby
   x86_64-darwin
   x86_64-linux-gnu
   x86_64-linux-musl
diff --git a/features/support/env.rb b/features/support/env.rb
index d695049db..bf3dac0a6 100644
--- a/features/support/env.rb
+++ b/features/support/env.rb
@@ -86,7 +86,7 @@ def default_drivers
 
 default_drivers
 
-Capybara.server = :webrick
+Capybara.server = :puma, { Silent: true }
 Capybara.default_max_wait_time = 15
 # Capybara.ignore_hidden_elements = false
 

From 8cc697b9be5f56c29435c20d4f46f50eb8c705c0 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Thu, 9 Jul 2026 14:34:17 -0400
Subject: [PATCH 09/14] remove data_elements from saved patient

---
 lib/cypress/scoop_and_filter.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 3fc58efb1..52eb6abd2 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -26,15 +26,17 @@ def scoop_and_filter(patient)
       # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset.
       # This array stores the cloned entries to be added
       multi_vs_negation_elements = []
-      patient.qdmPatient.dataElements.keep_if { |de| data_element_used_by_measure(de) }
+      de_to_delete = []
+      de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless data_element_used_by_measure(de) }
       patient.qdmPatient.dataElements.each do |data_element|
         scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient)
         data_element.dataElementCodes.first
       end
       # keep data element if codes is not empty
-      patient.qdmPatient.dataElements.keep_if { |data_element| data_element.dataElementCodes.present? }
+      de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless de.dataElementCodes.present? }
       patient.qdmPatient.dataElements.concat(demographic_criteria)
       patient.qdmPatient.dataElements.concat(multi_vs_negation_elements)
+      de_to_delete.compact.each(&:destroy)
       patient
     end
 

From 14bffb29ab409e3823b1a9beeba6d54b5a41b608 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Thu, 9 Jul 2026 15:43:59 -0400
Subject: [PATCH 10/14] ignore demographics in the scoop and filter

---
 lib/cypress/scoop_and_filter.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 52eb6abd2..7a23c4d6f 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -22,7 +22,6 @@ def get_non_demographic_category_statuses(measures)
     end
 
     def scoop_and_filter(patient)
-      demographic_criteria = patient.qdmPatient.dataElements.collect { |de| de if de.qdmCategory == 'patient_characteristic' }.compact
       # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset.
       # This array stores the cloned entries to be added
       multi_vs_negation_elements = []
@@ -34,7 +33,6 @@ def scoop_and_filter(patient)
       end
       # keep data element if codes is not empty
       de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless de.dataElementCodes.present? }
-      patient.qdmPatient.dataElements.concat(demographic_criteria)
       patient.qdmPatient.dataElements.concat(multi_vs_negation_elements)
       de_to_delete.compact.each(&:destroy)
       patient
@@ -45,6 +43,8 @@ def scoop_and_filter(patient)
     # Method to remove codes from a data element that are not relevant to measure.
     # Multi_vs_negation_elements is an array of cloned elements to add to patient record to capture all of the negated valuesets
     def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient)
+      return if data_element.qdmCategory == 'patient_characteristic'
+
       # keep if data_element code and codesystem is in one of the relevant_codes
       # Also keep all negated valuesets, we'll deal with those later
       data_element.dataElementCodes.keep_if do |de_code|
@@ -100,6 +100,8 @@ def data_element_category_and_status(data_element)
 
     # returns true if a patients data element is used by a measure
     def data_element_used_by_measure(data_element)
+      return true if data_element.qdmCategory == 'patient_characteristic'
+
       !@de_category_statuses_for_measures.index do |dcs|
         dcs[:category] == data_element['qdmCategory'] && dcs[:status] == data_element['qdmStatus']
       end.nil?

From 01bbd1bd4e758c1cdba84b4d9cd9a4854b1e9546 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Thu, 9 Jul 2026 16:38:28 -0400
Subject: [PATCH 11/14] break up scoop and filter from negation replacement

---
 .rubocop.yml                        |  1 +
 lib/cypress/patient_zipper.rb       |  2 +-
 lib/cypress/population_clone_job.rb |  4 +--
 lib/cypress/scoop_and_filter.rb     | 52 ++++++++++++++++-------------
 4 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/.rubocop.yml b/.rubocop.yml
index 9bdc4405c..61f928a44 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -105,6 +105,7 @@ Metrics/ClassLength:
     - 'lib/cypress/demographics_randomizer.rb'
     - 'lib/cypress/expected_results_calculator.rb'
     - 'lib/cypress/population_clone_job.rb'
+    - 'lib/cypress/scoop_and_filter.rb'
     - 'lib/validators/cat3_population_validator.rb'
 Metrics/CyclomaticComplexity:
   Max: 12
diff --git a/lib/cypress/patient_zipper.rb b/lib/cypress/patient_zipper.rb
index 54a262ac4..9f6ddc102 100644
--- a/lib/cypress/patient_zipper.rb
+++ b/lib/cypress/patient_zipper.rb
@@ -79,7 +79,7 @@ def self.zip(file, patients, format)
           sf_patient.id = patient.id
           # CMS529 requires that entries point to the ecounters that they are related to
           sf_patient.add_encounter_ids_to_events if measures.map(&:hqmf_id).intersect?(APP_CONSTANTS['result_measures'].map(&:hqmf_id))
-          patient_scoop_and_filter.scoop_and_filter(sf_patient)
+          patient_scoop_and_filter.replace_negated_codes_with_valueset(sf_patient)
           z.put_next_entry("#{next_entry_path(patient, i)}.#{FORMAT_EXTENSIONS[format.to_sym]}")
           z << formatter.export(sf_patient)
         end
diff --git a/lib/cypress/population_clone_job.rb b/lib/cypress/population_clone_job.rb
index 180e4195a..235a77d8d 100644
--- a/lib/cypress/population_clone_job.rb
+++ b/lib/cypress/population_clone_job.rb
@@ -47,7 +47,7 @@ def perform
         end
       end
 
-      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures, for_calculation: true) if @options['scoop_and_filter']
+      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures) if @options['scoop_and_filter']
 
       # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different
       # providers for each patient
@@ -98,7 +98,7 @@ def clone_and_save_patient(patient, prng, provider = nil, allow_dups: false)
         original_patient_id: patient.id,
         correlation_id: options['test_id']
       }
-      @patient_scoop_and_filter&.scoop_and_filter(cloned_patient)
+      @patient_scoop_and_filter&.scoop_and_filter(cloned_patient, replace_negations: false, persist_scoop: true)
       unnumerify cloned_patient if patient.givenNames.map { |n| n =~ /\d/ }.any? || patient.familyName =~ /\d/
       cloned_patient.medical_record_number = next_medical_record_number unless options['disable_randomization']
       @test.reload
diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 7a23c4d6f..239bf2be8 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -2,8 +2,7 @@
 
 module Cypress
   class ScoopAndFilter
-    def initialize(measures, for_calculation: false)
-      @for_calculation = for_calculation
+    def initialize(measures)
       @valuesets = measures.collect(&:value_sets).flatten.uniq
       @relevant_codes = codes_in_measures
       @de_category_statuses_for_measures = get_non_demographic_category_statuses(measures)
@@ -21,20 +20,39 @@ def get_non_demographic_category_statuses(measures)
       end.flatten.uniq.compact
     end
 
-    def scoop_and_filter(patient)
-      # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset.
-      # This array stores the cloned entries to be added
-      multi_vs_negation_elements = []
+    def scoop_and_filter(patient, replace_negations: true, persist_scoop: false)
       de_to_delete = []
       de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless data_element_used_by_measure(de) }
       patient.qdmPatient.dataElements.each do |data_element|
-        scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient)
-        data_element.dataElementCodes.first
+        scoop_and_filter_data_element_codes(data_element)
       end
       # keep data element if codes is not empty
       de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless de.dataElementCodes.present? }
+      if persist_scoop
+        de_to_delete.compact.each(&:destroy)
+      else
+        ids_to_delete = de_to_delete.compact.map(&:id)
+        patient.qdmPatient.dataElements.keep_if { |data_element| !ids_to_delete.include?(data_element.id) }
+      end
+      replace_negated_codes_with_valueset(patient) if replace_negations
+      patient
+    end
+
+    def replace_negated_codes_with_valueset(patient)
+      # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset.
+      # This array stores the cloned entries to be added
+      multi_vs_negation_elements = []
+      patient.qdmPatient.dataElements.each do |data_element|
+        next unless data_element.respond_to?('negationRationale') && data_element.negationRationale
+
+        replace_negated_code_with_valueset(data_element, multi_vs_negation_elements)
+        next if data_element.dataElementCodes.blank?
+
+        # add data element valueset and other potentially relevant valueset descriptions
+        codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
+        Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
+      end
       patient.qdmPatient.dataElements.concat(multi_vs_negation_elements)
-      de_to_delete.compact.each(&:destroy)
       patient
     end
 
@@ -42,7 +60,7 @@ def scoop_and_filter(patient)
 
     # Method to remove codes from a data element that are not relevant to measure.
     # Multi_vs_negation_elements is an array of cloned elements to add to patient record to capture all of the negated valuesets
-    def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient)
+    def scoop_and_filter_data_element_codes(data_element)
       return if data_element.qdmCategory == 'patient_characteristic'
 
       # keep if data_element code and codesystem is in one of the relevant_codes
@@ -55,16 +73,6 @@ def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements
 
       scoop_and_filter_data_element_fields(data_element)
       remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
-      # Return if all codes and valuesets have been removed
-      return if data_element.dataElementCodes.blank?
-      return unless data_element.respond_to?('negationRationale') && data_element.negationRationale
-
-      replace_negated_code_with_valueset(data_element, multi_vs_negation_elements) unless @for_calculation
-      return if data_element.dataElementCodes.blank?
-
-      # add data element valueset and other potentially relevant valueset descriptions
-      codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
-      Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
     end
 
     def scoop_and_filter_data_element_fields(data_element)
@@ -107,7 +115,6 @@ def data_element_used_by_measure(data_element)
       end.nil?
     end
 
-    # rubocop:disable Metrics/AbcSize
     def remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
       de = data_element
       vsets = if de.codes.any? { |code| code.system == '1.2.3.4.5.6.7.8.9.10' }
@@ -122,9 +129,8 @@ def remove_irrelevant_valuesets_and_add_description_to_data_element(data_element
       end
 
       vs = vsets.first
-      de.description = vs.display_name unless @for_calculation
+      de.description = vs.display_name
     end
-    # rubocop:enable Metrics/AbcSize
 
     def value_set_appropriate_for_data_element(data_element, valueset_oid)
       @de_category_statuses_for_measures.include?(category: data_element['qdmCategory'], status: data_element['qdmStatus'], oid: valueset_oid)

From 59189b0e62fd9239f7d030b209b4fb37993c31cd Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Thu, 9 Jul 2026 16:47:08 -0400
Subject: [PATCH 12/14] patient zipper no longer does the scoop and filter. It
 relies on the patient record has already been filtered

---
 test/unit/lib/patient_zipper_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/lib/patient_zipper_test.rb b/test/unit/lib/patient_zipper_test.rb
index fae7e2652..f1c87bbc0 100644
--- a/test/unit/lib/patient_zipper_test.rb
+++ b/test/unit/lib/patient_zipper_test.rb
@@ -73,7 +73,7 @@ class PatientZipperTest < ActiveSupport::TestCase
         doc.root.add_namespace_definition('cda', 'urn:hl7-org:v3')
         doc.root.add_namespace_definition('sdtc', 'urn:hl7-org:sdtc')
         # There should be 1 entry with encounter ids. The fixture measures do not have PhysicalExamPerformed criteria, so that entry will be filtered out
-        assert_equal 1, doc.xpath('//sdtc:templateId[@root="2.16.840.1.113883.10.20.24.3.150"]').size, 'There should be 1 entry with encounter ids'
+        assert_equal 2, doc.xpath('//sdtc:templateId[@root="2.16.840.1.113883.10.20.24.3.150"]').size, 'There should be 1 entry with encounter ids'
         count += 1
       end
     end

From 3b095440bd85a8e20b2954645f0aabd5975b0703 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Fri, 10 Jul 2026 10:47:56 -0400
Subject: [PATCH 13/14] data elements with blank negated valuesets should also
 be removed from export

---
 lib/cypress/scoop_and_filter.rb      | 7 ++++++-
 test/models/cat1_filter_task_test.rb | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 239bf2be8..593b433a1 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -42,17 +42,22 @@ def replace_negated_codes_with_valueset(patient)
       # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset.
       # This array stores the cloned entries to be added
       multi_vs_negation_elements = []
+      ids_to_delete = []
       patient.qdmPatient.dataElements.each do |data_element|
         next unless data_element.respond_to?('negationRationale') && data_element.negationRationale
 
         replace_negated_code_with_valueset(data_element, multi_vs_negation_elements)
-        next if data_element.dataElementCodes.blank?
+        if data_element.dataElementCodes.blank?
+          ids_to_delete << data_element.id
+          next
+        end
 
         # add data element valueset and other potentially relevant valueset descriptions
         codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
         Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
       end
       patient.qdmPatient.dataElements.concat(multi_vs_negation_elements)
+      patient.qdmPatient.dataElements.keep_if { |data_element| !ids_to_delete.include?(data_element.id) }
       patient
     end
 
diff --git a/test/models/cat1_filter_task_test.rb b/test/models/cat1_filter_task_test.rb
index 6f919c70e..6cc638179 100644
--- a/test/models/cat1_filter_task_test.rb
+++ b/test/models/cat1_filter_task_test.rb
@@ -22,7 +22,7 @@ def test_task_good_results_should_pass
     perform_enqueued_jobs do
       te = task.execute(testfile, user)
       te.reload
-      assert_empty te.execution_errors, 'test execution with known good results should have no errors'
+      assert_empty te.execution_errors.only_errors, 'test execution with known good results should have no errors'
     end
   end
 end

From 539cea9b47d0391fef7e9a250d973cd5ca910203 Mon Sep 17 00:00:00 2001
From: David Czulada <dczulada@mitre.org>
Date: Tue, 14 Jul 2026 08:51:32 -0400
Subject: [PATCH 14/14] don't save data element descriptions. they are measure
 dependent

---
 lib/cypress/scoop_and_filter.rb | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb
index 593b433a1..2d7a6396b 100644
--- a/lib/cypress/scoop_and_filter.rb
+++ b/lib/cypress/scoop_and_filter.rb
@@ -24,7 +24,7 @@ def scoop_and_filter(patient, replace_negations: true, persist_scoop: false)
       de_to_delete = []
       de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless data_element_used_by_measure(de) }
       patient.qdmPatient.dataElements.each do |data_element|
-        scoop_and_filter_data_element_codes(data_element)
+        scoop_and_filter_data_element_codes(data_element, persist_scoop)
       end
       # keep data element if codes is not empty
       de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless de.dataElementCodes.present? }
@@ -65,7 +65,7 @@ def replace_negated_codes_with_valueset(patient)
 
     # Method to remove codes from a data element that are not relevant to measure.
     # Multi_vs_negation_elements is an array of cloned elements to add to patient record to capture all of the negated valuesets
-    def scoop_and_filter_data_element_codes(data_element)
+    def scoop_and_filter_data_element_codes(data_element, persist_scoop)
       return if data_element.qdmCategory == 'patient_characteristic'
 
       # keep if data_element code and codesystem is in one of the relevant_codes
@@ -77,7 +77,8 @@ def scoop_and_filter_data_element_codes(data_element)
       return if data_element.dataElementCodes.blank?
 
       scoop_and_filter_data_element_fields(data_element)
-      remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
+      # For repeatability, don't do this if you are saving the record
+      remove_irrelevant_valuesets_and_add_description_to_data_element(data_element) unless persist_scoop
     end
 
     def scoop_and_filter_data_element_fields(data_element)