From 629ce2fda6e45b4e881a708dbb34cacafe01747e Mon Sep 17 00:00:00 2001 From: Trevor White <108891102+tdub29@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:41:48 -0700 Subject: [PATCH 1/2] feat(mlb): provenance sidecars for new outputs write_provenance() records git commit and dirty files, R and package versions, platform, command, named env vars and sha256 of inputs and outputs next to each output; check_provenance() refuses a cached artifact built under a different recipe. Frozen scripts are untouched; new experiments call it. Rscript research/r/mlb/provenance.R runs a self-check. --- research/r/mlb/provenance.R | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 research/r/mlb/provenance.R diff --git a/research/r/mlb/provenance.R b/research/r/mlb/provenance.R new file mode 100644 index 0000000..4fab3e5 --- /dev/null +++ b/research/r/mlb/provenance.R @@ -0,0 +1,54 @@ +# Provenance sidecars for new outputs (the reproducibility contract in the 2026-10 review): what built a +# file, from which inputs, under which settings. The frozen scripts do not call this; new experiments do. +# +# source("research/r/mlb/provenance.R") +# write_provenance("data/mlb/x/predictions.csv", inputs = "data/mlb/matchup/features.rds", env = c("OUT_TAG", "ASOF_LAG")) +# check_provenance("data/mlb/x/predictions.csv", inputs = "data/mlb/matchup/features.rds", env = c("OUT_TAG", "ASOF_LAG")) +# +# write_provenance() writes .provenance.json: git commit and dirty files under research/r/mlb, R and +# attached package versions, platform, command line, the named env vars as set ("" when unset), and sha256 +# of every input and output. check_provenance() stops when a named env var or input hash differs from the +# run that built the file, so a cached artifact is not reused under another recipe. +# +# Rscript research/r/mlb/provenance.R # self-check in a temp dir + +sha256_of <- function(f) c(as.character(openssl::sha256(file(f)))) # c() drops the "hash" class + +write_provenance <- function(outputs, inputs = character(), env = character()) { + git <- function(...) tryCatch(system2("git", c(...), stdout = TRUE, stderr = FALSE), error = function(e) NA_character_) + si <- utils::sessionInfo() + meta <- list( + written_utc = format(Sys.time(), "%Y-%m-%dT%H:%M:%SZ", tz = "UTC"), + git_commit = git("rev-parse", "HEAD"), + git_dirty = git("status", "--porcelain", "--", "research/r/mlb"), + r_version = R.version.string, platform = R.version$platform, + packages = lapply(si$otherPkgs, function(p) p$Version), + command = commandArgs(FALSE), + env = as.list(Sys.getenv(env, unset = "", names = TRUE)), + inputs = as.list(vapply(inputs, sha256_of, "")), + outputs = as.list(vapply(outputs, sha256_of, ""))) + for (o in outputs) jsonlite::write_json(meta, paste0(o, ".provenance.json"), auto_unbox = TRUE, pretty = TRUE) + invisible(meta) +} + +check_provenance <- function(output, inputs = character(), env = character()) { + p <- paste0(output, ".provenance.json") + if (!file.exists(p)) stop("no provenance sidecar for ", output) + old <- jsonlite::read_json(p, simplifyVector = TRUE) + now <- Sys.getenv(env, unset = "", names = TRUE) + for (n in env) if (!identical(as.character(old$env[[n]]), now[[n]])) + stop("env ", n, " is '", now[[n]], "' but ", output, " was built with '", old$env[[n]], "'") + for (f in inputs) if (!identical(as.character(old$inputs[[f]]), sha256_of(f))) + stop("input ", f, " changed since ", output, " was built") + invisible(TRUE) +} + +if (sys.nframe() == 0L) { # Rscript provenance.R + d <- tempfile(); dir.create(d); i <- file.path(d, "in.csv"); o <- file.path(d, "out.csv") + writeLines("a", i); writeLines("b", o) + Sys.setenv(PROV_TEST = "1"); write_provenance(o, inputs = i, env = "PROV_TEST") + stopifnot(check_provenance(o, inputs = i, env = "PROV_TEST")) + Sys.setenv(PROV_TEST = "2"); stopifnot(inherits(try(check_provenance(o, env = "PROV_TEST"), silent = TRUE), "try-error")) + writeLines("c", i); stopifnot(inherits(try(check_provenance(o, inputs = i), silent = TRUE), "try-error")) + cat("provenance self-check ok\n") +} From 17b8fb475f6c7f03c381d0a90d96124302e8a15e Mon Sep 17 00:00:00 2001 From: Trevor White <108891102+tdub29@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:48:02 -0700 Subject: [PATCH 2/2] test(mlb): provenance in the testthat suite; CI installs jsonlite and openssl --- .github/workflows/ci.yml | 2 ++ research/r/mlb/provenance.R | 11 ---------- research/r/tests/testthat.R | 1 + research/r/tests/testthat/test-provenance.R | 23 +++++++++++++++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 research/r/tests/testthat/test-provenance.R diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 446cc0f..3481c06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,5 +79,7 @@ jobs: any::testthat any::tseries any::ggplot2 + any::jsonlite + any::openssl - name: testthat run: Rscript research/r/tests/testthat.R diff --git a/research/r/mlb/provenance.R b/research/r/mlb/provenance.R index 4fab3e5..672335f 100644 --- a/research/r/mlb/provenance.R +++ b/research/r/mlb/provenance.R @@ -9,8 +9,6 @@ # attached package versions, platform, command line, the named env vars as set ("" when unset), and sha256 # of every input and output. check_provenance() stops when a named env var or input hash differs from the # run that built the file, so a cached artifact is not reused under another recipe. -# -# Rscript research/r/mlb/provenance.R # self-check in a temp dir sha256_of <- function(f) c(as.character(openssl::sha256(file(f)))) # c() drops the "hash" class @@ -43,12 +41,3 @@ check_provenance <- function(output, inputs = character(), env = character()) { invisible(TRUE) } -if (sys.nframe() == 0L) { # Rscript provenance.R - d <- tempfile(); dir.create(d); i <- file.path(d, "in.csv"); o <- file.path(d, "out.csv") - writeLines("a", i); writeLines("b", o) - Sys.setenv(PROV_TEST = "1"); write_provenance(o, inputs = i, env = "PROV_TEST") - stopifnot(check_provenance(o, inputs = i, env = "PROV_TEST")) - Sys.setenv(PROV_TEST = "2"); stopifnot(inherits(try(check_provenance(o, env = "PROV_TEST"), silent = TRUE), "try-error")) - writeLines("c", i); stopifnot(inherits(try(check_provenance(o, inputs = i), silent = TRUE), "try-error")) - cat("provenance self-check ok\n") -} diff --git a/research/r/tests/testthat.R b/research/r/tests/testthat.R index 23c73d8..b95eb31 100644 --- a/research/r/tests/testthat.R +++ b/research/r/tests/testthat.R @@ -15,5 +15,6 @@ source(file.path(root, "mlb", "features.R")) source(file.path(root, "mlb", "model.R")) source(file.path(root, "mlb", "evaluate.R")) source(file.path(root, "mlb", "windows.R")) +source(file.path(root, "mlb", "provenance.R")) test_dir(file.path(root, "tests", "testthat"), stop_on_failure = TRUE) diff --git a/research/r/tests/testthat/test-provenance.R b/research/r/tests/testthat/test-provenance.R new file mode 100644 index 0000000..b9e7ca5 --- /dev/null +++ b/research/r/tests/testthat/test-provenance.R @@ -0,0 +1,23 @@ +# Provenance sidecars: a cached output is refused once its recipe or inputs change. + +test_that("a sidecar round-trips and catches env and input changes", { + d <- tempfile(); dir.create(d); on.exit(unlink(d, recursive = TRUE)) + i <- file.path(d, "in.csv"); o <- file.path(d, "out.csv") + writeLines("a", i); writeLines("b", o) + old <- Sys.getenv("PROV_TEST", unset = NA); on.exit(if (is.na(old)) Sys.unsetenv("PROV_TEST") else Sys.setenv(PROV_TEST = old), add = TRUE) + + Sys.setenv(PROV_TEST = "1") + write_provenance(o, inputs = i, env = "PROV_TEST") + expect_true(file.exists(paste0(o, ".provenance.json"))) + expect_true(check_provenance(o, inputs = i, env = "PROV_TEST")) + + Sys.setenv(PROV_TEST = "2") + expect_error(check_provenance(o, env = "PROV_TEST"), "PROV_TEST") + + writeLines("c", i) + expect_error(check_provenance(o, inputs = i), "changed") +}) + +test_that("a missing sidecar is an error, not a pass", { + expect_error(check_provenance(tempfile()), "no provenance sidecar") +})