diff --git a/.rubocop.yml b/.rubocop.yml index 9bdc4405c..61f928a44 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -105,6 +105,7 @@ Metrics/ClassLength: - 'lib/cypress/demographics_randomizer.rb' - 'lib/cypress/expected_results_calculator.rb' - 'lib/cypress/population_clone_job.rb' + - 'lib/cypress/scoop_and_filter.rb' - 'lib/validators/cat3_population_validator.rb' Metrics/CyclomaticComplexity: Max: 12 diff --git a/Gemfile b/Gemfile index c5ffd6b65..4b0781be2 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ source 'https://rubygems.org' ruby '3.4.9' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 8.1.0' +gem 'rails', '~> 8.0.0', '>= 8.0.2.1' gem 'propshaft' @@ -126,7 +126,6 @@ group :development, :test do # remove scss_lint, incompatible with sass dependency upgrades # gem 'scss_lint', require: false gem 'selenium-webdriver' - gem 'webrick' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 570f6d443..76bbe419d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,31 +1,29 @@ GEM remote: https://rubygems.org/ specs: - action_text-trix (2.1.19) - railties - actioncable (8.1.3) - actionpack (= 8.1.3) - activesupport (= 8.1.3) + actioncable (8.0.5) + actionpack (= 8.0.5) + activesupport (= 8.0.5) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (8.1.3) - actionpack (= 8.1.3) - activejob (= 8.1.3) - activerecord (= 8.1.3) - activestorage (= 8.1.3) - activesupport (= 8.1.3) + actionmailbox (8.0.5) + actionpack (= 8.0.5) + activejob (= 8.0.5) + activerecord (= 8.0.5) + activestorage (= 8.0.5) + activesupport (= 8.0.5) mail (>= 2.8.0) - actionmailer (8.1.3) - actionpack (= 8.1.3) - actionview (= 8.1.3) - activejob (= 8.1.3) - activesupport (= 8.1.3) + actionmailer (8.0.5) + actionpack (= 8.0.5) + actionview (= 8.0.5) + activejob (= 8.0.5) + activesupport (= 8.0.5) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (8.1.3) - actionview (= 8.1.3) - activesupport (= 8.1.3) + actionpack (8.0.5) + actionview (= 8.0.5) + activesupport (= 8.0.5) nokogiri (>= 1.8.5) rack (>= 2.2.4) rack-session (>= 1.0.1) @@ -33,43 +31,42 @@ GEM rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (8.1.3) - action_text-trix (~> 2.1.15) - actionpack (= 8.1.3) - activerecord (= 8.1.3) - activestorage (= 8.1.3) - activesupport (= 8.1.3) + actiontext (8.0.5) + actionpack (= 8.0.5) + activerecord (= 8.0.5) + activestorage (= 8.0.5) + activesupport (= 8.0.5) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (8.1.3) - activesupport (= 8.1.3) + actionview (8.0.5) + activesupport (= 8.0.5) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (8.1.3) - activesupport (= 8.1.3) + activejob (8.0.5) + activesupport (= 8.0.5) globalid (>= 0.3.6) - activemodel (8.1.3) - activesupport (= 8.1.3) - activerecord (8.1.3) - activemodel (= 8.1.3) - activesupport (= 8.1.3) + activemodel (8.0.5) + activesupport (= 8.0.5) + activerecord (8.0.5) + activemodel (= 8.0.5) + activesupport (= 8.0.5) timeout (>= 0.4.0) - activestorage (8.1.3) - actionpack (= 8.1.3) - activejob (= 8.1.3) - activerecord (= 8.1.3) - activesupport (= 8.1.3) + activestorage (8.0.5) + actionpack (= 8.0.5) + activejob (= 8.0.5) + activerecord (= 8.0.5) + activesupport (= 8.0.5) marcel (~> 1.0) - activesupport (8.1.3) + activesupport (8.0.5) base64 + benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) - json logger (>= 1.4.2) minitest (>= 5.1) securerandom (>= 0.3) @@ -265,7 +262,6 @@ GEM railties (>= 6.1.0) faker (3.8.0) i18n (>= 1.8.11, < 2) - ffi (1.17.4) ffi (1.17.4-aarch64-linux-gnu) ffi (1.17.4-aarch64-linux-musl) ffi (1.17.4-arm-linux-gnu) @@ -333,7 +329,7 @@ GEM thor (>= 0.14, < 2.0) jquery-ui-rails (8.0.0) railties (>= 3.2.16) - json (2.19.4) + json (2.20.0) kaminari-core (1.2.2) kaminari-mongoid (1.0.2) kaminari-core (~> 1.0) @@ -373,11 +369,10 @@ GEM mini_magick (5.3.1) logger mini_mime (1.1.5) - mini_portile2 (2.8.9) minitest (5.27.0) - minitest-rails (8.1.0) + minitest-rails (8.0.0) minitest (~> 5.20) - railties (>= 8.1.0, < 8.2.0) + railties (>= 8.0.0, < 8.1.0) minitest-reporters (1.8.0) ansi builder @@ -415,9 +410,6 @@ GEM newrelic_rpm (10.4.0) logger nio4r (2.7.5) - nokogiri (1.19.4) - mini_portile2 (~> 2.8.2) - racc (~> 1.4) nokogiri (1.19.4-aarch64-linux-gnu) racc (~> 1.4) nokogiri (1.19.4-aarch64-linux-musl) @@ -477,20 +469,20 @@ GEM rack (>= 1.3) rackup (2.3.1) rack (>= 3) - rails (8.1.3) - actioncable (= 8.1.3) - actionmailbox (= 8.1.3) - actionmailer (= 8.1.3) - actionpack (= 8.1.3) - actiontext (= 8.1.3) - actionview (= 8.1.3) - activejob (= 8.1.3) - activemodel (= 8.1.3) - activerecord (= 8.1.3) - activestorage (= 8.1.3) - activesupport (= 8.1.3) + rails (8.0.5) + actioncable (= 8.0.5) + actionmailbox (= 8.0.5) + actionmailer (= 8.0.5) + actionpack (= 8.0.5) + actiontext (= 8.0.5) + actionview (= 8.0.5) + activejob (= 8.0.5) + activemodel (= 8.0.5) + activerecord (= 8.0.5) + activestorage (= 8.0.5) + activesupport (= 8.0.5) bundler (>= 1.15.0) - railties (= 8.1.3) + railties (= 8.0.5) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) @@ -512,9 +504,9 @@ GEM ostruct require_all (~> 3.0) ruby-progressbar - railties (8.1.3) - actionpack (= 8.1.3) - activesupport (= 8.1.3) + railties (8.0.5) + actionpack (= 8.0.5) + activesupport (= 8.0.5) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -578,9 +570,6 @@ GEM logger ruby2_keywords (0.0.5) rubyzip (1.3.0) - sass-embedded (1.99.0) - google-protobuf (~> 4.31) - rake (>= 13) sass-embedded (1.99.0-aarch64-linux-gnu) google-protobuf (~> 4.31) sass-embedded (1.99.0-aarch64-linux-musl) @@ -669,7 +658,6 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.9.2) websocket (1.2.11) websocket-driver (0.8.2) base64 @@ -689,7 +677,6 @@ PLATFORMS arm-linux-musl arm-linux-musleabihf arm64-darwin - ruby x86_64-darwin x86_64-linux-gnu x86_64-linux-musl @@ -747,7 +734,7 @@ DEPENDENCIES pry pry-nav puma (~> 7.2.1) - rails (~> 8.1.0) + rails (~> 8.0.0, >= 8.0.2.1) rails-controller-testing rails-perftest rails_best_practices @@ -771,7 +758,6 @@ DEPENDENCIES vmstat web-console (~> 4.2.0) webmock - webrick RUBY VERSION ruby 3.4.9p82 diff --git a/app/models/product_test.rb b/app/models/product_test.rb index 0c9c2b426..f9f046dda 100644 --- a/app/models/product_test.rb +++ b/app/models/product_test.rb @@ -92,11 +92,13 @@ def generate_patients(job_id = nil) if product.randomize_patients # If we're using a "slim test deck", don't pass in any random IDs random_ids = slim_test_deck? ? [] : gather_patient_ids - Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'randomization_ids' => random_ids, + Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, + 'scoop_and_filter' => true, 'randomization_ids' => random_ids, 'include_virtual' => (measure_ids & APP_CONSTANTS['telehealth_ineligible_measures']), 'randomize_demographics' => true, 'generate_provider' => product.c4_test, 'job_id' => job_id).perform else - Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'disable_randomization' => true).perform + Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, + 'scoop_and_filter' => true, 'disable_randomization' => true).perform end end diff --git a/features/support/env.rb b/features/support/env.rb index d695049db..bf3dac0a6 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -86,7 +86,7 @@ def default_drivers default_drivers -Capybara.server = :webrick +Capybara.server = :puma, { Silent: true } Capybara.default_max_wait_time = 15 # Capybara.ignore_hidden_elements = false diff --git a/lib/cypress/patient_zipper.rb b/lib/cypress/patient_zipper.rb index 54a262ac4..9f6ddc102 100644 --- a/lib/cypress/patient_zipper.rb +++ b/lib/cypress/patient_zipper.rb @@ -79,7 +79,7 @@ def self.zip(file, patients, format) sf_patient.id = patient.id # CMS529 requires that entries point to the ecounters that they are related to sf_patient.add_encounter_ids_to_events if measures.map(&:hqmf_id).intersect?(APP_CONSTANTS['result_measures'].map(&:hqmf_id)) - patient_scoop_and_filter.scoop_and_filter(sf_patient) + patient_scoop_and_filter.replace_negated_codes_with_valueset(sf_patient) z.put_next_entry("#{next_entry_path(patient, i)}.#{FORMAT_EXTENSIONS[format.to_sym]}") z << formatter.export(sf_patient) end diff --git a/lib/cypress/population_clone_job.rb b/lib/cypress/population_clone_job.rb index a6b5d2db7..235a77d8d 100644 --- a/lib/cypress/population_clone_job.rb +++ b/lib/cypress/population_clone_job.rb @@ -47,6 +47,8 @@ def perform end end + @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures) if @options['scoop_and_filter'] + # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different # providers for each patient provider = @test.provider if @test.instance_of?(MeasureTest) @@ -96,6 +98,7 @@ def clone_and_save_patient(patient, prng, provider = nil, allow_dups: false) original_patient_id: patient.id, correlation_id: options['test_id'] } + @patient_scoop_and_filter&.scoop_and_filter(cloned_patient, replace_negations: false, persist_scoop: true) unnumerify cloned_patient if patient.givenNames.map { |n| n =~ /\d/ }.any? || patient.familyName =~ /\d/ cloned_patient.medical_record_number = next_medical_record_number unless options['disable_randomization'] @test.reload diff --git a/lib/cypress/scoop_and_filter.rb b/lib/cypress/scoop_and_filter.rb index 580fd4937..593b433a1 100644 --- a/lib/cypress/scoop_and_filter.rb +++ b/lib/cypress/scoop_and_filter.rb @@ -20,20 +20,44 @@ def get_non_demographic_category_statuses(measures) end.flatten.uniq.compact end - def scoop_and_filter(patient) - demographic_criteria = patient.qdmPatient.dataElements.collect { |de| de if de.qdmCategory == 'patient_characteristic' }.compact + def scoop_and_filter(patient, replace_negations: true, persist_scoop: false) + de_to_delete = [] + de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless data_element_used_by_measure(de) } + patient.qdmPatient.dataElements.each do |data_element| + scoop_and_filter_data_element_codes(data_element) + end + # keep data element if codes is not empty + de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless de.dataElementCodes.present? } + if persist_scoop + de_to_delete.compact.each(&:destroy) + else + ids_to_delete = de_to_delete.compact.map(&:id) + patient.qdmPatient.dataElements.keep_if { |data_element| !ids_to_delete.include?(data_element.id) } + end + replace_negated_codes_with_valueset(patient) if replace_negations + patient + end + + def replace_negated_codes_with_valueset(patient) # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset. # This array stores the cloned entries to be added multi_vs_negation_elements = [] - patient.qdmPatient.dataElements.keep_if { |de| data_element_used_by_measure(de) } + ids_to_delete = [] patient.qdmPatient.dataElements.each do |data_element| - scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient) - data_element.dataElementCodes.first + next unless data_element.respond_to?('negationRationale') && data_element.negationRationale + + replace_negated_code_with_valueset(data_element, multi_vs_negation_elements) + if data_element.dataElementCodes.blank? + ids_to_delete << data_element.id + next + end + + # add data element valueset and other potentially relevant valueset descriptions + codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" } + Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle) end - # keep data element if codes is not empty - patient.qdmPatient.dataElements.keep_if { |data_element| data_element.dataElementCodes.present? } - patient.qdmPatient.dataElements.concat(demographic_criteria) patient.qdmPatient.dataElements.concat(multi_vs_negation_elements) + patient.qdmPatient.dataElements.keep_if { |data_element| !ids_to_delete.include?(data_element.id) } patient end @@ -41,26 +65,46 @@ def scoop_and_filter(patient) # Method to remove codes from a data element that are not relevant to measure. # Multi_vs_negation_elements is an array of cloned elements to add to patient record to capture all of the negated valuesets - def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient) + def scoop_and_filter_data_element_codes(data_element) + return if data_element.qdmCategory == 'patient_characteristic' + # keep if data_element code and codesystem is in one of the relevant_codes # Also keep all negated valuesets, we'll deal with those later data_element.dataElementCodes.keep_if do |de_code| - @relevant_codes.include?(code: de_code.code, system: de_code.system) || de_code.system == '1.2.3.4.5.6.7.8.9.10' + relevant_code?(de_code) || de_code.system == '1.2.3.4.5.6.7.8.9.10' end # Return if all codes have been removed return if data_element.dataElementCodes.blank? + scoop_and_filter_data_element_fields(data_element) remove_irrelevant_valuesets_and_add_description_to_data_element(data_element) - # Return if all codes and valuesets have been removed - return if data_element.dataElementCodes.blank? - return unless data_element.respond_to?('negationRationale') && data_element.negationRationale + end - replace_negated_code_with_valueset(data_element, multi_vs_negation_elements) - return if data_element.dataElementCodes.blank? + def scoop_and_filter_data_element_fields(data_element) + # Iterate through each field to see coded fields include relevant codes + data_element.fields.keys.each do |field_name| + next if data_element[field_name].nil? + + # Dianoses and Facility Locations are unique because they are arrays + if field_name == 'diagnoses' + data_element.diagnoses.keep_if do |diagnosis| + relevant_code?(diagnosis.code) + end + data_element.diagnoses = nil if data_element.diagnoses.blank? + elsif field_name == 'facilityLocations' + data_element.facilityLocations.keep_if do |facility_location| + relevant_code?(facility_location.code) + end + data_element.facilityLocations = nil if data_element.facilityLocations.blank? + end + next unless data_element.fields[field_name].type == QDM::Code + + data_element[field_name] = nil unless relevant_code?(data_element[field_name]) + end + end - # add data element valueset and other potentially relevant valueset descriptions - codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" } - Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle) + def relevant_code?(code) + @relevant_codes.include?(code: code.code, system: code.system) end def data_element_category_and_status(data_element) @@ -69,6 +113,8 @@ def data_element_category_and_status(data_element) # returns true if a patients data element is used by a measure def data_element_used_by_measure(data_element) + return true if data_element.qdmCategory == 'patient_characteristic' + !@de_category_statuses_for_measures.index do |dcs| dcs[:category] == data_element['qdmCategory'] && dcs[:status] == data_element['qdmStatus'] end.nil? diff --git a/test/models/cat1_filter_task_test.rb b/test/models/cat1_filter_task_test.rb index 6f919c70e..6cc638179 100644 --- a/test/models/cat1_filter_task_test.rb +++ b/test/models/cat1_filter_task_test.rb @@ -22,7 +22,7 @@ def test_task_good_results_should_pass perform_enqueued_jobs do te = task.execute(testfile, user) te.reload - assert_empty te.execution_errors, 'test execution with known good results should have no errors' + assert_empty te.execution_errors.only_errors, 'test execution with known good results should have no errors' end end end diff --git a/test/unit/lib/patient_zipper_test.rb b/test/unit/lib/patient_zipper_test.rb index fae7e2652..f1c87bbc0 100644 --- a/test/unit/lib/patient_zipper_test.rb +++ b/test/unit/lib/patient_zipper_test.rb @@ -73,7 +73,7 @@ class PatientZipperTest < ActiveSupport::TestCase doc.root.add_namespace_definition('cda', 'urn:hl7-org:v3') doc.root.add_namespace_definition('sdtc', 'urn:hl7-org:sdtc') # There should be 1 entry with encounter ids. The fixture measures do not have PhysicalExamPerformed criteria, so that entry will be filtered out - assert_equal 1, doc.xpath('//sdtc:templateId[@root="2.16.840.1.113883.10.20.24.3.150"]').size, 'There should be 1 entry with encounter ids' + assert_equal 2, doc.xpath('//sdtc:templateId[@root="2.16.840.1.113883.10.20.24.3.150"]').size, 'There should be 1 entry with encounter ids' count += 1 end end