From 6d1c1d574b755a4a2394d9d74d038860fe89f2b6 Mon Sep 17 00:00:00 2001 From: "Joshua Uhalt, Ph.D." Date: Sun, 27 Sep 2026 10:52:06 -0400 Subject: [PATCH 1/2] Separate the Delphi caveats from the teaching, fit anova_content() in 80 columns, and show every guide in the site menu Three of the four items left for 1.0 after 0.9.0. No computed value changes. The package is a teaching tool for instructors and students and a tool for researchers' own work, so the output teaches without burying the result. - Delphi print, in two layers. Two short caveats always print: read kappa as a trend beside the share who kept their rating, not against a cut-off; and with fewer than about 40 experts the kappa intervals cover less than their stated 95%. The fuller explanation (the weights, the intraclass-correlation equivalence, why kappa falls as a panel converges, Klar et al.'s coverage figures) moves into the key under "How the stability statistic works", where students see it and a researcher can hide it. The alternative methods' critiques still always print, as the method-options policy requires. The example goes from 100 lines to 92, or 58 with the key hidden. A test checks both layers. - anova_content() prints within 80 columns (79 on the example), dropping the competitor column that htd() and the rating workflow show and that stays in strongest_competitor; the note says so. A test checks the width. - The website's Articles menu lists every guide under a heading, in course order (Start here, Workflow guides, Planning and reporting, After content review); the workflow guides had been hidden under "More articles". The auxiliary helpers' reference label no longer calls itself "not flagship v0.1.0 workflows". - ROADMAP: the schema-v1 check against solomonR is recorded as withdrawn; its maintainer made solomonR a separate project that reads no handoffs (2026-09-25). Version 0.9.0.9000. Co-Authored-By: Claude Opus 5.5 --- DESCRIPTION | 2 +- NEWS.md | 24 +++++++ R/component_print.R | 11 +-- R/delphi_validity.R | 96 +++++++++++++++++---------- ROADMAP.md | 7 +- _pkgdown.yml | 21 ++++-- tests/testthat/test-component-print.R | 4 ++ tests/testthat/test-delphi-validity.R | 27 ++++++++ vignettes/delphi-rounds.Rmd | 5 +- 9 files changed, 147 insertions(+), 50 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d7a5f6d..c6c45b3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: contentvalidR Type: Package Title: Tools for Substantive and Content Validity Pretesting -Version: 0.9.0 +Version: 0.9.0.9000 Authors@R: c( person(given = "Joshua", family = "Uhalt", email = "Josh.Uhalt@gmail.com", role = c("aut", "cre")) diff --git a/NEWS.md b/NEWS.md index 0146282..704485d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,27 @@ +# contentvalidR 0.9.0.9000 (development version) + +## Finishing touches before 1.0 (in development) + +No computed value changes. + +* **The Delphi printout separates what changes a reading from what teaches + it.** Two short caveats always print: read kappa as a trend beside the + share of experts who kept their rating, not against a cut-off; and with + fewer than about 40 experts the kappa intervals cover less than their + stated 95%. The fuller explanation (the weights, why kappa falls as a panel + converges, Klar et al.'s coverage figures) now sits in the key under "How + the stability statistic works", so an instructor's students see it and a + researcher can hide it with `options(contentvalidR.show_key = FALSE)`. The + critique printed for an alternative stability method still always prints. + The example printout goes from 100 lines to 92, or 58 with the key hidden. +* `anova_content()` prints within 80 columns. The strongest competitor, which + `htd()` and the construct-rating workflow already show, stays in the + `strongest_competitor` column. +* The website's Articles menu lists every guide under a heading, in the order + a course would use them: Start here, Workflow guides, Planning and + reporting, and After content review. The workflow guides were hidden under + "More articles". + # contentvalidR 0.9.0 Ninth public release. v0.9.0 is about how the package presents itself before diff --git a/R/component_print.R b/R/component_print.R index a895936..b80a1c6 100644 --- a/R/component_print.R +++ b/R/component_print.R @@ -153,12 +153,13 @@ print.contentvalid_anova <- function(x, digits = 2, ...) { sprintf("F(%s, %s) = %s", .fmt(d1, digits, bounded = FALSE), .fmt(d2, digits, bounded = FALSE), .fmt(x$F, digits, bounded = FALSE))) + # Narrow enough for an 80-column console; the strongest competitor + # stays in the `strongest_competitor` column. data.frame(item = x$item, target = x$target, judges = x$n_complete, - `omnibus test` = test, p = .fmt_p(x$p_screen), + `F test` = test, p = .fmt_p(x$p_screen), `partial eta^2` = .fmt(x$partial_eta2, digits), - competitor = x$strongest_competitor, `contrast p` = .fmt_p(x$max_contrast_p), - `contrasts met` = .yes_no(x$contrast_pass), + met = .yes_no(x$contrast_pass), stringsAsFactors = FALSE, check.names = FALSE) }, notes = c( @@ -166,7 +167,9 @@ print.contentvalid_anova <- function(x, digits = 2, ...) { "Within-judge omnibus tests are Greenhouse-Geisser corrected, so their degrees of freedom are fractional." }, paste("contrast p: the largest p among the planned target-versus-other", - "contrasts; attr(x, \"contrasts\") holds every contrast.") + "contrasts; met: whether every one of them met the screening", + "criterion. attr(x, \"contrasts\") holds each contrast, and", + "`strongest_competitor` the construct rated closest to the target.") ) ) } diff --git a/R/delphi_validity.R b/R/delphi_validity.R index a30c5ee..883cfe4 100644 --- a/R/delphi_validity.R +++ b/R/delphi_validity.R @@ -229,42 +229,61 @@ list(data = d, rounds = round_levels) } +# What the printout says about the stability method, in two layers. `always` +# holds what changes how a result should be read, and prints every time. +# `teaching` holds the fuller explanation, printed with the key, so an +# experienced reader can hide it (options(contentvalidR.show_key = FALSE)) +# without losing a caveat. Each element is a paragraph. .delphi_method_note <- function(method, weights, intervals = TRUE) { + if (identical(method, "kappa")) { + return(list( + always = c( + paste("Read kappa as a trend across rounds, beside the share of experts", + "who kept their rating (unchanged), not against a cut-off: kappa", + "falls as a panel converges on one category, so a stable panel", + "can show a low kappa (Holey et al., 2007)."), + if (intervals) { + paste("The kappa intervals resample the experts (Klar et al., 2002).", + "With fewer than about 40 experts they cover less than their", + "stated 95%, so read them as rough indications of precision,", + "not as tests.") + } + ), + teaching = c( + paste( + "Stability is weighted kappa between each expert's ratings in", + "consecutive rounds (Holey et al., 2007), with", weights, "weights.", + if (weights == "quadratic") { + paste("A change of two scale points counts four times a change of", + "one. With these weights kappa equals the intraclass", + "correlation of the two rounds' ratings, so a shift of the", + "whole panel counts as instability (Fleiss & Cohen, 1973).") + } else { + paste("A change of two scale points counts twice a change of one", + "(Cohen, 1968).") + }, + "No verbal labels such as 'substantial' are shown, because kappa", + "falls when ratings converge, which is what a Delphi aims for: Holey", + "et al. saw a low kappa for their most-agreed statement." + ), + if (intervals) { + paste("The intervals are percentile bootstraps that resample the", + "experts, the units the two rounds cross-classify: the", + "procedure Klar et al. (2002) describe for kappa. They", + "evaluated it for an unweighted kappa on two categories, and a", + "nominal 95% interval covered about 83% of the time with 20", + "units and 91% with 30, reaching 94% only from 40 up.") + } + ) + )) + } + # The alternatives are contested, so each one's critique always prints. + list(always = .delphi_alternative_note(method), teaching = NULL) +} + +.delphi_alternative_note <- function(method) { switch( method, - # Each element is a paragraph; the print method separates them. - kappa = c( - paste( - "Stability is weighted kappa between each expert's ratings in", - "consecutive rounds (Holey et al., 2007), with", weights, "weights.", - if (weights == "quadratic") { - paste("A change of two scale points counts four times a change of", - "one. With these weights kappa equals the intraclass", - "correlation of the two rounds' ratings, so a shift of the", - "whole panel counts as instability (Fleiss & Cohen, 1973).") - } else { - paste("A change of two scale points counts twice a change of one", - "(Cohen, 1968).") - }, - "Read kappa as a trend across rounds, not against a cut-off. No verbal", - "labels such as 'substantial' are shown: kappa falls when ratings", - "converge on one category, which is what a Delphi aims for, so a panel", - "whose experts nearly all kept their answer can still show a low", - "kappa. Holey et al. saw this for their most-agreed statement. Read", - "kappa next to the share of experts who kept their rating (unchanged)." - ), - if (intervals) { - paste("The intervals are percentile bootstraps that resample the", - "experts, which are the units the two rounds cross-classify.", - "That is the procedure Klar et al. (2002) describe for kappa,", - "but they evaluated it for an unweighted kappa on two", - "categories and found a nominal 95% interval covered about 83%", - "of the time with 20 units and 91% with 30, reaching 94% only", - "from 40 up. Most panels are smaller than that, so the interval", - "is narrower than its label claims: read it as indicative of", - "precision, not as a test.") - } - ), lambda = paste( "Stability is lambda, the index of predictive association that Chaffin", "and Talley (1980) take from Goodman and Kruskal: how much knowing an", @@ -864,9 +883,9 @@ print.contentvalid_delphi <- function(x, digits = 2, ...) { cat("\n") note <- .delphi_method_note(s$stability, s$kappa_weights, intervals = any(!is.na(stab$lower))) - for (i in seq_along(note)) { + for (i in seq_along(note$always)) { if (i > 1L) cat("\n") - .say(note[[i]]) + .say(note$always[[i]]) } if (is.null(s$consensus_threshold)) { cat("\n") @@ -885,6 +904,13 @@ print.contentvalid_delphi <- function(x, digits = 2, ...) { } if (.show_key()) { + if (length(note$teaching)) { + cat("\nHow the stability statistic works\n") + for (i in seq_along(note$teaching)) { + if (i > 1L) cat("\n") + .say(note$teaching[[i]], indent = 2L, exdent = 2L) + } + } key <- c("prop_agree", "prop_unchanged", switch(s$stability, kappa = "kappa_w", lambda = "lambda", chisq_individual = "chi_sq_individual", diff --git a/ROADMAP.md b/ROADMAP.md index 15c93d2..fa5abf5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -691,9 +691,12 @@ settled, and the joint walkthrough. removed; `posthoc_pass` is documented as deprecated, starting its clock. - [x] Freeze handoff schema version 1 ([#54](https://github.com/JUhalt/contentvalidR/issues/54)), after - checking it against nomologR and solomonR. Adds `keying`, + checking it against nomologR, its reader. Adds `keying`, `response_min`, and `response_max`, which come from the analyst and - never from the fit. + never from the fit. solomonR was also asked at the time; on + 2026-09-25 its maintainer made solomonR a separate project that reads + no handoffs, and withdrew the issue that recorded its answer, so it is + not a consumer of schema version 1. - [x] The joint walkthrough on shared teaching data ([#55](https://github.com/JUhalt/contentvalidR/issues/55)): `vignette("one-item-set-both-stages")`, where the two stages disagree in diff --git a/_pkgdown.yml b/_pkgdown.yml index f0d4863..9f811ec 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -79,7 +79,7 @@ reference: - similarity_from_sort - title: Auxiliary and compatibility helpers - desc: Retained utilities that are not flagship v0.1.0 workflows. + desc: Kept for older analyses and sensitivity checks; not recommended workflows. contents: - qfactor_content - signal_detection @@ -94,18 +94,27 @@ reference: articles: - title: Start here - navbar: ~ + navbar: Start here contents: - getting-started - reading-output - - design-and-reporting - - reporting-examples - - handoff-to-empirical-validation - - one-item-set-both-stages - title: Workflow guides + navbar: Workflow guides contents: - item-sort-validity - construct-rating-validity - expert-panel-validity - delphi-rounds + +- title: Planning and reporting + navbar: Planning and reporting + contents: + - design-and-reporting + - reporting-examples + +- title: After content review + navbar: After content review + contents: + - handoff-to-empirical-validation + - one-item-set-both-stages diff --git a/tests/testthat/test-component-print.R b/tests/testthat/test-component-print.R index 6c92d65..1e8f562 100644 --- a/tests/testthat/test-component-print.R +++ b/tests/testthat/test-component-print.R @@ -113,6 +113,10 @@ test_that("the construct-rating components print as APA tables", { within <- shown(anova_content(d)) expect_match(within, "Greenhouse-Geisser corrected", fixed = TRUE) + # The table fits an 80-column console. + old_width <- options(width = 80) + on.exit(options(old_width), add = TRUE) + expect_lte(max(nchar(utils::capture.output(print(anova_content(d))))), 80L) expect_match(within, "F[(][0-9.]+, [0-9.]+[)] = [0-9.]+") # A between-judges design has whole-number degrees of freedom and no # Greenhouse-Geisser note. diff --git a/tests/testthat/test-delphi-validity.R b/tests/testthat/test-delphi-validity.R index 9346eec..1424b0a 100644 --- a/tests/testthat/test-delphi-validity.R +++ b/tests/testthat/test-delphi-validity.R @@ -350,3 +350,30 @@ test_that("the round fits feed compare_rounds, and reporting works", { rep <- content_report(fit, format = "data.frame") expect_true(all(c("item", "prop_agree", "prop_unchanged", "stability") %in% names(rep))) }) + +test_that("the kappa caveats always print, and the teaching layer prints with the key", { + fit <- delphi_validity(small_delphi(), lo = 1, hi = 4, B = 100, seed = 3) + text <- function() { + gsub("[[:space:]]+", " ", paste(capture.output(print(fit)), collapse = " ")) + } + with_key <- text() + expect_match(with_key, "Read kappa as a trend across rounds", fixed = TRUE) + expect_match(with_key, "fewer than about 40 experts", fixed = TRUE) + expect_match(with_key, "How the stability statistic works", fixed = TRUE) + expect_match(with_key, "covered about 83% of the time with 20 units", fixed = TRUE) + + old <- options(contentvalidR.show_key = FALSE) + on.exit(options(old), add = TRUE) + without <- text() + # What changes how a result is read survives hiding the key ... + expect_match(without, "Read kappa as a trend across rounds", fixed = TRUE) + expect_match(without, "fewer than about 40 experts", fixed = TRUE) + # ... and the teaching layer goes with it. + expect_false(grepl("How the stability statistic works", without, fixed = TRUE)) + expect_false(grepl("intraclass correlation", without, fixed = TRUE)) + # An alternative method's critique is never hidden. + alt <- paste(capture.output(print(delphi_validity(small_delphi(), lo = 1, hi = 4, + stability = "lambda", B = 0))), + collapse = " ") + expect_match(alt, "predictability, not agreement", fixed = TRUE) +}) diff --git a/vignettes/delphi-rounds.Rmd b/vignettes/delphi-rounds.Rmd index 167e7af..3d901ce 100644 --- a/vignettes/delphi-rounds.Rmd +++ b/vignettes/delphi-rounds.Rmd @@ -100,8 +100,9 @@ fit <- delphi_validity(ratings, lo = 1, hi = 4, consensus_threshold = 0.75, fit ``` -The printout ends with a key to every column and status. The rest of this -article turns the key off, to keep the output short: +The printout ends with a key: how the stability statistic works, and what each +column and decision means. The two caveats above the key print either way. The +rest of this article turns the key off, to keep the output short: ```{r key-off} options(contentvalidR.show_key = FALSE) From 05c5c110376c812d33c2b52ead4275088850bf57 Mon Sep 17 00:00:00 2001 From: "Joshua Uhalt, Ph.D." Date: Sun, 27 Sep 2026 11:47:11 -0400 Subject: [PATCH 2/2] Cite Scheibe et al. from the 2002 edition, and record the nomologR partnership The package's percent-change rule was checked against the editors' 2002 web edition of Linstone and Turoff, so the reference now names that edition, with the chapter's pages (257-281), its link, and "(Original work published 1975)". In-text citations read "Scheibe et al. (1975/2002)", as APA 7 asks for a republished work, including the citation a handoff carries for stability = "percent_change". ROADMAP.md now states the maintainer's intent: contentvalidR and nomologR are partners, kept on the same page about handoffs before and after 1.0, and solomonR is a separate project that reads no handoffs and takes no part in the joint 1.0. Co-Authored-By: Claude Opus 5.5 --- NEWS.md | 7 +++++++ R/content_handoff.R | 2 +- R/delphi_validity.R | 16 +++++++++------- README.Rmd | 2 +- README.md | 6 ++++-- ROADMAP.md | 5 +++++ inst/REFERENCES.bib | 8 +++++--- man/delphi_validity.Rd | 8 +++++--- vignettes/delphi-rounds.Rmd | 12 +++++++----- 9 files changed, 44 insertions(+), 22 deletions(-) diff --git a/NEWS.md b/NEWS.md index 704485d..975720a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -21,6 +21,13 @@ No computed value changes. a course would use them: Start here, Workflow guides, Planning and reporting, and After content review. The workflow guides were hidden under "More articles". +* **Scheibe et al. is cited from the edition the package's method was checked + against:** the editors' 2002 web edition of Linstone and Turoff, with the + chapter's pages (257–281), a link, and "(Original work published 1975)", as + APA 7 requires for a republished work. In-text citations read + "Scheibe et al. (1975/2002)", including the citation a handoff carries for + `stability = "percent_change"`. The 1975 printing's pages could not be + checked. # contentvalidR 0.9.0 diff --git a/R/content_handoff.R b/R/content_handoff.R index ab383b9..42b852a 100644 --- a/R/content_handoff.R +++ b/R/content_handoff.R @@ -244,7 +244,7 @@ lambda = "Chaffin & Talley (1980)", chisq_individual = "Chaffin & Talley (1980)", chisq_group = "Dajani, Sincoff & Talley (1979)", - percent_change = "Scheibe, Skutsch & Schofer (1975)" + percent_change = "Scheibe et al. (1975/2002)" ) # Relevance evidence comes from each item's own last round, so the interval diff --git a/R/delphi_validity.R b/R/delphi_validity.R index 883cfe4..f2fbf8f 100644 --- a/R/delphi_validity.R +++ b/R/delphi_validity.R @@ -7,7 +7,7 @@ .delphi_stability_methods <- c("kappa", "lambda", "chisq_individual", "chisq_group", "percent_change") -# Scheibe, Skutsch & Schofer (1975): change below 15% is read as stable. +# Scheibe et al. (1975/2002): change below 15% is read as stable. .delphi_scheibe_cut <- 0.15 .delphi_disagreement_weights <- function(k, weights) { @@ -60,7 +60,7 @@ n_rows = nrow(O), n_cols = ncol(O)) } -# Scheibe et al. (1975): net person-changes between the two rounds' histograms +# Scheibe et al. (1975/2002): net person-changes between the two rounds' histograms # as a share of the experts compared. .delphi_percent_change <- function(x1, x2, k) { sum(abs(tabulate(x2, k) - tabulate(x1, k))) / 2 / length(x1) @@ -313,7 +313,7 @@ "category. Expected counts below 5 are flagged." ), percent_change = paste( - "Stability is Scheibe, Skutsch and Schofer's (1975) net change: half the", + "Stability is the net change of Scheibe et al. (1975/2002): half the", "summed differences between the two rounds' rating distributions, as a", "share of the experts compared, with change below 15% read as stable.", "The authors say the measure has no statistical theory behind it; the", @@ -370,7 +370,7 @@ #' * `"chisq_group"`: Dajani, Sincoff and Talley's (1979) chi-square test on #' the two rounds' distributions; a non-significant result is read as #' stable. -#' * `"percent_change"`: Scheibe, Skutsch and Schofer's (1975) net change, +#' * `"percent_change"`: the net change of Scheibe et al. (1975/2002), #' stable below 15%. #' #' The alternatives are published but contested, so the printed output @@ -502,9 +502,11 @@ #' Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement #' for categorical data. *Biometrics, 33*(1), 159–174. \doi{10.2307/2529310} #' -#' Scheibe, M., Skutsch, M., & Schofer, J. (1975). Experiments in Delphi +#' Scheibe, M., Skutsch, M., & Schofer, J. (2002). Experiments in Delphi #' methodology. In H. A. Linstone & M. Turoff (Eds.), *The Delphi method: -#' Techniques and applications*. Addison-Wesley. +#' Techniques and applications* (pp. 257–281). +#' \url{https://www.foresight.pl/assets/downloads/publications/Turoff_Linstone.pdf} +#' (Original work published 1975) #' #' @seealso [expert_validity()] for a single round, and [compare_rounds()], #' which accepts the fits in `details$round_fits`. @@ -796,7 +798,7 @@ print.contentvalid_delphi <- function(x, digits = 2, ...) { lambda = "lambda (Chaffin & Talley, 1980)", chisq_individual = "individual chi-square (Chaffin & Talley, 1980)", chisq_group = "group chi-square (Dajani et al., 1979)", - percent_change = "net percent change (Scheibe et al., 1975)" + percent_change = "net percent change (Scheibe et al., 1975/2002)" ) .say("Stability:", method_label, "between consecutive rounds") if (s$stability %in% c("chisq_individual", "chisq_group")) { diff --git a/README.Rmd b/README.Rmd index 20b9e14..59736c1 100644 --- a/README.Rmd +++ b/README.Rmd @@ -177,7 +177,7 @@ Works cited in this README, the help pages, and the vignettes. - Polit, D. F., & Beck, C. T. (2006). The content validity index: Are you sure you know what's being reported? Critique and recommendations. *Research in Nursing & Health, 29*(5), 489–497. https://doi.org/10.1002/nur.20147 - Polit, D. F., Beck, C. T., & Owen, S. V. (2007). Is the CVI an acceptable indicator of content validity? Appraisal and recommendations. *Research in Nursing & Health, 30*(4), 459–467. https://doi.org/10.1002/nur.20199 - Rovinelli, R. J., & Hambleton, R. K. (1977). On the use of content specialists in the assessment of criterion-referenced test item validity. *Dutch Journal of Educational Research, 2*, 49–60. -- Scheibe, M., Skutsch, M., & Schofer, J. (1975). Experiments in Delphi methodology. In H. A. Linstone & M. Turoff (Eds.), *The Delphi method: Techniques and applications*. Addison-Wesley. +- Scheibe, M., Skutsch, M., & Schofer, J. (2002). Experiments in Delphi methodology. In H. A. Linstone & M. Turoff (Eds.), *The Delphi method: Techniques and applications* (pp. 257–281). https://www.foresight.pl/assets/downloads/publications/Turoff_Linstone.pdf (Original work published 1975) - Schriesheim, C. A., Cogliser, C. C., Scandura, T. A., Lankau, M. J., & Powers, K. J. (1999). An empirical comparison of approaches for quantitatively assessing the content adequacy of paper-and-pencil measurement instruments. *Organizational Research Methods, 2*(2), 140–156. https://doi.org/10.1177/109442819922002 - Schriesheim, C. A., Powers, K. J., Scandura, T. A., Gardiner, C. C., & Lankau, M. J. (1993). Improving construct measurement in management research: Comments and a quantitative approach for assessing the theoretical content adequacy of paper-and-pencil survey-type instruments. *Journal of Management, 19*(2), 385–417. https://doi.org/10.1177/014920639301900208 - Sireci, S. G. (1998). The construct of content validity. *Social Indicators Research, 45*(1–3), 83–117. https://doi.org/10.1023/A:1006985528729 diff --git a/README.md b/README.md index a5ed1a6..70789fe 100644 --- a/README.md +++ b/README.md @@ -396,9 +396,11 @@ Works cited in this README, the help pages, and the vignettes. - Rovinelli, R. J., & Hambleton, R. K. (1977). On the use of content specialists in the assessment of criterion-referenced test item validity. *Dutch Journal of Educational Research, 2*, 49–60. -- Scheibe, M., Skutsch, M., & Schofer, J. (1975). Experiments in Delphi +- Scheibe, M., Skutsch, M., & Schofer, J. (2002). Experiments in Delphi methodology. In H. A. Linstone & M. Turoff (Eds.), *The Delphi method: - Techniques and applications*. Addison-Wesley. + Techniques and applications* (pp. 257–281). + + (Original work published 1975) - Schriesheim, C. A., Cogliser, C. C., Scandura, T. A., Lankau, M. J., & Powers, K. J. (1999). An empirical comparison of approaches for quantitatively assessing the content adequacy of paper-and-pencil diff --git a/ROADMAP.md b/ROADMAP.md index fa5abf5..879d8c4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -792,6 +792,11 @@ review, then empirical validation. They are joined by the handoff schema on both to release 1.0.0 together, so the version number tells users the pair is stable together. +The maintainer's intent (2026-09-27): contentvalidR and nomologR are partners, +kept on the same page about handoffs before and after 1.0. solomonR, the +maintainer's package for Solomon four-group designs, is a separate project: it +reads no handoffs and takes no part in the joint 1.0. + - Converge, don't move in lockstep. Minor versions stay independent until 1.0.0, and neither package tags 1.0.0 on its own. - After 1.0.0, major versions move together whenever the shared contract diff --git a/inst/REFERENCES.bib b/inst/REFERENCES.bib index 53a222e..7a68da0 100644 --- a/inst/REFERENCES.bib +++ b/inst/REFERENCES.bib @@ -410,13 +410,15 @@ @article{rovinelli1977 pages = {49--60} } -@incollection{scheibe1975, +@incollection{scheibe2002, author = {Scheibe, M. and Skutsch, M. and Schofer, J.}, - year = {1975}, + year = {2002}, title = {{Experiments in Delphi methodology}}, booktitle = {{The Delphi method: Techniques and applications}}, editor = {Linstone, H. A. and Turoff, M.}, - publisher = {Addison-Wesley} + pages = {257--281}, + note = {Original work published 1975}, + url = {https://www.foresight.pl/assets/downloads/publications/Turoff_Linstone.pdf} } @article{schriesheim1999, diff --git a/man/delphi_validity.Rd b/man/delphi_validity.Rd index 2c0100e..d0ca337 100644 --- a/man/delphi_validity.Rd +++ b/man/delphi_validity.Rd @@ -100,7 +100,7 @@ each expert's pair of ratings; a significant result is read as stable. \item \code{"chisq_group"}: Dajani, Sincoff and Talley's (1979) chi-square test on the two rounds' distributions; a non-significant result is read as stable. -\item \code{"percent_change"}: Scheibe, Skutsch and Schofer's (1975) net change, +\item \code{"percent_change"}: the net change of Scheibe et al. (1975/2002), stable below 15\%. } @@ -233,9 +233,11 @@ Royal Statistical Society: Series D (The Statistician), 51}(4), 467–478. Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement for categorical data. \emph{Biometrics, 33}(1), 159–174. \doi{10.2307/2529310} -Scheibe, M., Skutsch, M., & Schofer, J. (1975). Experiments in Delphi +Scheibe, M., Skutsch, M., & Schofer, J. (2002). Experiments in Delphi methodology. In H. A. Linstone & M. Turoff (Eds.), \emph{The Delphi method: -Techniques and applications}. Addison-Wesley. +Techniques and applications} (pp. 257–281). +\url{https://www.foresight.pl/assets/downloads/publications/Turoff_Linstone.pdf} +(Original work published 1975) } \seealso{ \code{\link[=expert_validity]{expert_validity()}} for a single round, and \code{\link[=compare_rounds]{compare_rounds()}}, diff --git a/vignettes/delphi-rounds.Rmd b/vignettes/delphi-rounds.Rmd index 3d901ce..aae0e43 100644 --- a/vignettes/delphi-rounds.Rmd +++ b/vignettes/delphi-rounds.Rmd @@ -199,8 +199,8 @@ no verbal labels. ## Why the default looks at individual experts -S6 is the reason stability is measured expert by expert. Under Scheibe, -Skutsch and Schofer's (1975) 15% rule, which compares the two rounds' overall +S6 is the reason stability is measured expert by expert. Under the 15% rule +of Scheibe et al. (1975/2002), which compares the two rounds' overall rating distributions, S6 looks perfectly stable: ```{r percent-change} @@ -289,7 +289,7 @@ a downstream package reads it. Consensus is agreement among these experts. It is not, by itself, evidence that an item is valid. Feedback between rounds can also press experts toward -agreement they do not hold: Scheibe, Skutsch and Schofer (1975) observed that +agreement they do not hold: Scheibe et al. (1975/2002) observed that participants who conformed most strongly were the least satisfied with the process. A `No consensus` result is not an instruction to drop an item, and `Consensus` is not an instruction to keep one. Read these results together @@ -329,6 +329,8 @@ https://doi.org/10.1186/1471-2288-7-52 Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement for categorical data. *Biometrics, 33*(1), 159–174. https://doi.org/10.2307/2529310 -Scheibe, M., Skutsch, M., & Schofer, J. (1975). Experiments in Delphi +Scheibe, M., Skutsch, M., & Schofer, J. (2002). Experiments in Delphi methodology. In H. A. Linstone & M. Turoff (Eds.), *The Delphi method: -Techniques and applications*. Addison-Wesley. +Techniques and applications* (pp. 257–281). +https://www.foresight.pl/assets/downloads/publications/Turoff_Linstone.pdf +(Original work published 1975)