TreatmentPatterns unit test failures

[anthonysena]

Getting the following error when running TreatmentPatterns unit tests:

Connecting using SQLite driver
EXECUTING: TreatmentPatternsModule
Connecting using SQLite driver
Analysis '1' pathway construction failed: SQLite is not supported by CDMConnector. Please use duckdb instead.
ERROR: Module failed: 1 analysis(es) failed. See logs and per-analysis artifacts in '~/Strategus-TP/work_folder/TreatmentPatternsModule' (failed_pathway_runs.csv). Errors: 'Analysis '1' pathway construction failed: SQLite is not supported by CDMConnector. Please use duckdb instead.'

It appears that these tests should use duckdb based on the error above.

[anthonysena]

Tagging @mvankessel-EMC for some assistance here since the TreatmentPatterns tests we currently have on the develop branch appear to have broken upon release of TreatmentPatterns v3.1.2 and I'm unsure what has changed. In the reprex below, I've copied in the functions from https://github.com/OHDSI/Strategus/blob/develop/tests/testthat/helper-TreatmentPatterns.R with a change to force the use of duckdb when creating Eunomia.

Then I'm attempting to run the "single analysis" test (reference: https://github.com/OHDSI/Strategus/blob/develop/tests/testthat/test-TreatmentPatterns.R#L63) and that apparently fails due to no one in the treatment table for cohort "ViralSinusitis" which then causes the rest of the unit test to fail since none of the other results files are produced.

When I roll back to TreatmentPatterns v3.1.1 the tests pass so I'm unclear on what has changed that would cause some of our test failures in Strategus.

# Taken from tests/testthat/helper-TreatmentPatterns.R
ableToRunTreatmentPatterns <- function() {
  all(
    require("CirceR", character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE),
    require("CohortGenerator", character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE),
    require("DatabaseConnector", character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE),
    require("SqlRender", character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE),
    require("Eunomia", character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE)
  )
}

# NOTE: This function is used in the tests and in the package maintenance
# to create the settings which includes other cohorts. So naming this
# to allow for appending cohorts to an existing cohort set, if needed
appendTreatmentPatternsCohorts <- function(cohortDefinitionSet = CohortGenerator::createEmptyCohortDefinitionSet()) {
  cohortJsonFiles <- list.files(
    system.file(
      package = "TreatmentPatterns",
      "exampleCohorts"
    ),
    full.names = TRUE
  )

  # NOTE: If this function is called with a non-empty
  # cohortDefinitionSet then we want to append the new cohorts
  # and not create it from scratch
  cdsRows <- nrow(cohortDefinitionSet)
  for (i in seq_len(length(cohortJsonFiles))) {
    cohortJsonFileName <- cohortJsonFiles[i]
    cohortName <- tools::file_path_sans_ext(basename(cohortJsonFileName))
    cohortJson <- readChar(cohortJsonFileName, file.info(
      cohortJsonFileName
    )$size)

    cohortExpression <- CirceR::cohortExpressionFromJson(cohortJson)

    cohortSql <- CirceR::buildCohortQuery(
      cohortExpression,
      options = CirceR::createGenerateOptions(generateStats = FALSE)
    )
    cohortDefinitionSet <- rbind(
      cohortDefinitionSet,
      data.frame(
        cohortId = (cdsRows + i),
        cohortName = cohortName,
        sql = cohortSql,
        json = cohortJson,
        stringsAsFactors = FALSE
      )
    )
  }

  return(cohortDefinitionSet)
}

getTreatmentPatternsCohorts <- function(cohortDefinitionSet = CohortGenerator::createEmptyCohortDefinitionSet()) {
  # Select Viral Sinusitis Cohort
  targetCohorts <- cohortDefinitionSet |>
    dplyr::filter(cohortName == "ViralSinusitis") |>
    dplyr::select(cohortId, cohortName)

  # Select everything BUT Viral Sinusitis cohorts
  eventCohorts <- cohortDefinitionSet |>
    dplyr::filter(cohortName != "ViralSinusitis" & cohortName != "Death") |>
    dplyr::select(cohortId, cohortName)

  exitCohorts <- cohortDefinitionSet |>
    dplyr::filter(cohortName == "Death") |>
    dplyr::select(cohortId, cohortName)

  cohorts <- dplyr::bind_rows(
    targetCohorts |> dplyr::mutate(type = "target"),
    eventCohorts |> dplyr::mutate(type = "event"),
    exitCohorts |> dplyr::mutate(type = "exit")
  )
  return(cohorts)
}

generateCohortTable <- function() {
  if (ableToRunTreatmentPatterns()) {
    connectionDetails <- Eunomia::getEunomiaConnectionDetails(dbms = "duckdb")
    cohortTableName <- "cohort_table"
    resultSchema <- "main"
    cdmSchema <- "main"

    cohortsToCreate <- appendTreatmentPatternsCohorts()

    cohortTableNames <- CohortGenerator::getCohortTableNames(
      cohortTable = cohortTableName
    )

    CohortGenerator::createCohortTables(
      connectionDetails = connectionDetails,
      cohortDatabaseSchema = resultSchema,
      cohortTableNames = cohortTableNames
    )

    # Generate the cohorts
    cohortsGenerated <- CohortGenerator::generateCohortSet(
      connectionDetails = connectionDetails,
      cdmDatabaseSchema = cdmSchema,
      cohortDatabaseSchema = resultSchema,
      cohortTableNames = cohortTableNames,
      cohortDefinitionSet = cohortsToCreate
    )

    cohorts <- getTreatmentPatternsCohorts(cohortsToCreate)

    return(list(
      cohorts = cohorts,
      connectionDetails = connectionDetails,
      cohortTableName = cohortTableName,
      resultSchema = resultSchema,
      cdmSchema = cdmSchema
    ))
  } else {
    return(NULL)
  }
}

tempDir <- file.path(tempdir(), "Strategus-TP")
  testSettings <- generateCohortTable()
#> Connecting using DuckDB driver
#> Creating cohort tables
#> - Created table main.cohort_table
#> - Created table main.cohort_table
#> - Created table main.cohort_table_inclusion
#> - Created table main.cohort_table_inclusion_result
#> - Created table main.cohort_table_inclusion_stats
#> - Created table main.cohort_table_summary_stats
#> - Created table main.cohort_table_censor_stats
#> - Created table main.cohort_table_subset_attrition
#> - Created table main.cohort_table_checksum
#> Creating cohort tables took 0.44secs
#> Connecting using DuckDB driver
#> Initiating cluster consisting only of main thread
#> 1/8- Generating cohort: Acetaminophen (id = 1)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.13 secs
#> 2/8- Generating cohort: Amoxicillin (id = 2)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.101 secs
#> 3/8- Generating cohort: Aspirin (id = 3)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.102 secs
#> 4/8- Generating cohort: Clavulanate (id = 4)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.0981 secs
#> 5/8- Generating cohort: Death (id = 5)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.0473 secs
#> 6/8- Generating cohort: Doxylamine (id = 6)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.0928 secs
#> 7/8- Generating cohort: PenicillinV (id = 7)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  12%  |                                                                              |==========                                                            |  15%  |                                                                              |============                                                          |  18%  |                                                                              |==============                                                        |  21%  |                                                                              |================                                                      |  24%  |                                                                              |===================                                                   |  26%  |                                                                              |=====================                                                 |  29%  |                                                                              |=======================                                               |  32%  |                                                                              |=========================                                             |  35%  |                                                                              |===========================                                           |  38%  |                                                                              |=============================                                         |  41%  |                                                                              |===============================                                       |  44%  |                                                                              |=================================                                     |  47%  |                                                                              |===================================                                   |  50%  |                                                                              |=====================================                                 |  53%  |                                                                              |=======================================                               |  56%  |                                                                              |=========================================                             |  59%  |                                                                              |===========================================                           |  62%  |                                                                              |=============================================                         |  65%  |                                                                              |===============================================                       |  68%  |                                                                              |=================================================                     |  71%  |                                                                              |===================================================                   |  74%  |                                                                              |======================================================                |  76%  |                                                                              |========================================================              |  79%  |                                                                              |==========================================================            |  82%  |                                                                              |============================================================          |  85%  |                                                                              |==============================================================        |  88%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.103 secs
#> 8/8- Generating cohort: ViralSinusitis (id = 8)
#>   |                                                                              |                                                                      |   0%  |                                                                              |==                                                                    |   3%  |                                                                              |====                                                                  |   6%  |                                                                              |======                                                                |   9%  |                                                                              |========                                                              |  11%  |                                                                              |==========                                                            |  14%  |                                                                              |============                                                          |  17%  |                                                                              |==============                                                        |  20%  |                                                                              |================                                                      |  23%  |                                                                              |==================                                                    |  26%  |                                                                              |====================                                                  |  29%  |                                                                              |======================                                                |  31%  |                                                                              |========================                                              |  34%  |                                                                              |==========================                                            |  37%  |                                                                              |============================                                          |  40%  |                                                                              |==============================                                        |  43%  |                                                                              |================================                                      |  46%  |                                                                              |==================================                                    |  49%  |                                                                              |====================================                                  |  51%  |                                                                              |======================================                                |  54%  |                                                                              |========================================                              |  57%  |                                                                              |==========================================                            |  60%  |                                                                              |============================================                          |  63%  |                                                                              |==============================================                        |  66%  |                                                                              |================================================                      |  69%  |                                                                              |==================================================                    |  71%  |                                                                              |====================================================                  |  74%  |                                                                              |======================================================                |  77%  |                                                                              |========================================================              |  80%  |                                                                              |==========================================================            |  83%  |                                                                              |============================================================          |  86%  |                                                                              |==============================================================        |  89%  |                                                                              |================================================================      |  91%  |                                                                              |==================================================================    |  94%  |                                                                              |====================================================================  |  97%  |                                                                              |======================================================================| 100%
#> Executing SQL took 0.107 secs
#> Generating cohort set took 2.01 secs

  tp <- Strategus::TreatmentPatternsModule$new()

  modSpec <- tp$createModuleSpecifications(
    cohorts = testSettings$cohorts,
    startAnchor = "startDate",
    windowStart = 0,
    endAnchor = "endDate",
    windowEnd = 0,
    minEraDuration = 7,
    splitEventCohorts = NULL,
    splitTime = NULL,
    eraCollapseSize = 14,
    combinationWindow = 7,
    minPostCombinationDuration = 7,
    filterTreatments = "First",
    maxPathLength = 5
  )

  analysisSpecifications <- Strategus::createEmptyAnalysisSpecifications() |>
    Strategus::addTreatmentPatternsModuleSpecifications(modSpec)

  workFolder <- file.path(tempDir, "work_folder")

  executionSettings <- Strategus::createCdmExecutionSettings(
    workDatabaseSchema = testSettings$resultSchema,
    cdmDatabaseSchema = testSettings$cdmSchema,
    cohortTableNames = CohortGenerator::getCohortTableNames(cohortTable = "cohort_table"),
    tempEmulationSchema = NULL,
    workFolder = workFolder,
    resultsFolder = file.path(tempDir, "results_folder"),
    logFileName = "log.txt",
    minCellCount = 5,
    incremental = FALSE,
    maxCores = 1
  )

  Strategus::execute(
    connectionDetails = testSettings$connectionDetails,
    analysisSpecifications = analysisSpecifications,
    executionSettings = executionSettings
  )
#> Currently in a tryCatch or withCallingHandlers block, so unable to add global calling handlers. ParallelLogger will not capture R messages, errors, and warnings, only explicit calls to ParallelLogger. (This message will not be shown again this R session)
#> Collecting OMOP CDM Metadata
#> Connecting using DuckDB driverEXECUTING: TreatmentPatternsModuleConnecting using DuckDB driverInserting data took 0.0817 secsNote: method with signature 'DBIConnection#SQL' chosen for function 'dbQuoteIdentifier',
#>  target signature 'DatabaseConnectorDbiConnection#SQL'.
#>  "DatabaseConnectorConnection#character" would also be valid
#> -- Qualifying records for cohort definitions: 1, 2, 3, 4, 5, 6, 7, 8
#>  Records: 14041
#>  Subjects: 2693
#> -- Removing records < minEraDuration (7)
#>  Records: 0
#>  Subjects: 0
#> >> Starting on target: 8 (ViralSinusitis)
#> -- Removing events outside window (startDate: 0 | endDate: 0)
#>  Records: 0
#>  Subjects: 0
#> Warning in constructPathways(settings = args, andromeda = andromeda): No cases
#> found for: 8 (ViralSinusitis). Generating empty treatmentHistory table.
#> -- treatment construction done
#>  Records: 0
#>  Subjects: 0
#> Treatment History table is empty. Nothing to export.
#> Results available at: C:\Users\ADMIN_~3\AppData\Local\Temp\2\RtmpCMYnA2\Strategus-TP\results_folder/TreatmentPatternsModule
#> ── EXECUTION SUMMARY ───────────────────────────────────────────────────────────
#> ✔ TreatmentPatternsModule  (Execution Time: 11.1 secs)

^{Created on 2026-03-02 with reprex v2.1.1}