From d55eecd6931ed3386b59911f9a67dde512af7fe5 Mon Sep 17 00:00:00 2001 From: Yanjie Huang Date: Tue, 8 Sep 2026 23:15:44 -0700 Subject: [PATCH 1/2] Improve repository guides, reproducibility status and CPU checks --- .github/CITATION_SCHEMA.md | 8 + .github/ISSUE_TEMPLATE/bug_report.yml | 26 + .github/ISSUE_TEMPLATE/config.yml | 1 + .../citation-file-format-1.2.0.schema.json | 1882 +++++++++++++++++ .github/citation-schema.LICENSE | 395 ++++ .github/pull_request_template.md | 11 + .github/workflows/cpu.yml | 73 + .github/workflows/resources.yml | 42 + .gitignore | 1 + CHANGELOG.md | 12 +- CITATION.cff | 2 +- CONTRIBUTING.md | 45 + README.md | 931 ++------ docs/BENCHMARK_PROTOCOLS.md | 8 + docs/INSTALLATION.md | 132 ++ docs/OFFICIAL_MODEL_RESOURCES.md | 29 +- docs/PAPER_WORKFLOWS.md | 16 +- docs/RELEASE_STATUS.md | 100 +- docs/REPRODUCTION.md | 186 +- docs/RESOURCES.md | 77 + docs/REVIEWER_REQUIREMENTS.md | 28 + docs/USAGE.md | 453 ++++ pyproject.toml | 1 + reproduction/milena_14b/expected/README.md | 7 +- .../release/model_manifests/rnabert.json | 13 + .../release/model_manifests/rnafm.json | 13 + .../release/model_manifests/rnamsm.json | 13 + .../public_resource_metadata_20260909.json | 114 + .../recovered_competitor_assets_20260909.json | 121 ++ .../recovered_rna_snapshots_20260909.json | 136 ++ .../release/resource_access_20260909.json | 158 ++ reproduction/release/resource_links.json | 99 + scripts/check_installed_package.py | 43 + scripts/check_repository.py | 79 + scripts/check_resource_links.py | 61 + scripts/reproduce_milena.py | 16 +- tests/test_milena_entrypoint.py | 108 + 37 files changed, 4492 insertions(+), 948 deletions(-) create mode 100644 .github/CITATION_SCHEMA.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/citation-file-format-1.2.0.schema.json create mode 100644 .github/citation-schema.LICENSE create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/cpu.yml create mode 100644 .github/workflows/resources.yml create mode 100644 CONTRIBUTING.md create mode 100644 docs/INSTALLATION.md create mode 100644 docs/RESOURCES.md create mode 100644 docs/REVIEWER_REQUIREMENTS.md create mode 100644 docs/USAGE.md create mode 100644 reproduction/release/model_manifests/rnabert.json create mode 100644 reproduction/release/model_manifests/rnafm.json create mode 100644 reproduction/release/model_manifests/rnamsm.json create mode 100644 reproduction/release/public_resource_metadata_20260909.json create mode 100644 reproduction/release/recovered_competitor_assets_20260909.json create mode 100644 reproduction/release/recovered_rna_snapshots_20260909.json create mode 100644 reproduction/release/resource_access_20260909.json create mode 100644 reproduction/release/resource_links.json create mode 100644 scripts/check_installed_package.py create mode 100644 scripts/check_repository.py create mode 100644 scripts/check_resource_links.py diff --git a/.github/CITATION_SCHEMA.md b/.github/CITATION_SCHEMA.md new file mode 100644 index 0000000..5e31c4f --- /dev/null +++ b/.github/CITATION_SCHEMA.md @@ -0,0 +1,8 @@ +# Citation schema + +The bundled CFF 1.2.0 JSON Schema is from +https://github.com/citation-file-format/citation-file-format/blob/1.2.0/schema.json + +SHA256: `0b8d22140da702d766df318dcff3a91af2f39521298dcf36d76315fd99cc169b`. +The upstream license is retained in `citation-schema.LICENSE`. +It is bundled so citation validation does not require network access. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..d184da9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,26 @@ +name: Bug report +description: Report an installation, execution or reproducibility problem. +title: "[Bug] " +body: + - type: textarea + id: environment + attributes: + label: Version and environment + description: Git commit, Python, PyTorch, CUDA, OS, GPU if relevant, and installation method. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Command and input + description: Exact command and a small public or synthetic input that reproduces the problem. + render: shell + validations: + required: true + - type: textarea + id: error + attributes: + label: Actual and expected behavior + description: Include the error log; remove credentials and confidential data. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0086358 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/citation-file-format-1.2.0.schema.json b/.github/citation-file-format-1.2.0.schema.json new file mode 100644 index 0000000..762194b --- /dev/null +++ b/.github/citation-file-format-1.2.0.schema.json @@ -0,0 +1,1882 @@ +{ + "$id": "https://citation-file-format.github.io/1.2.0/schema.json", + "$schema": "http://json-schema.org/draft-07/schema", + "additionalProperties": false, + "definitions": { + "address": { + "description": "An address.", + "minLength": 1, + "type": "string" + }, + "alias": { + "description": "An alias.", + "minLength": 1, + "type": "string" + }, + "city": { + "description": "A city", + "minLength": 1, + "type": "string" + }, + "commit": { + "description": "The (e.g., Git) commit hash or (e.g., Subversion) revision number of the work.", + "minLength": 1, + "type": "string" + }, + "country": { + "$comment": "ISO 3166-1 alpha-2 codes can be found at https://en.wikipedia.org/wiki/ISO_3166-1", + "description": "The ISO 3166-1 alpha-2 country code for a country.", + "enum": [ + "AD", + "AE", + "AF", + "AG", + "AI", + "AL", + "AM", + "AO", + "AQ", + "AR", + "AS", + "AT", + "AU", + "AW", + "AX", + "AZ", + "BA", + "BB", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BL", + "BM", + "BN", + "BO", + "BQ", + "BR", + "BS", + "BT", + "BV", + "BW", + "BY", + "BZ", + "CA", + "CC", + "CD", + "CF", + "CG", + "CH", + "CI", + "CK", + "CL", + "CM", + "CN", + "CO", + "CR", + "CU", + "CV", + "CW", + "CX", + "CY", + "CZ", + "DE", + "DJ", + "DK", + "DM", + "DO", + "DZ", + "EC", + "EE", + "EG", + "EH", + "ER", + "ES", + "ET", + "FI", + "FJ", + "FK", + "FM", + "FO", + "FR", + "GA", + "GB", + "GD", + "GE", + "GF", + "GG", + "GH", + "GI", + "GL", + "GM", + "GN", + "GP", + "GQ", + "GR", + "GS", + "GT", + "GU", + "GW", + "GY", + "HK", + "HM", + "HN", + "HR", + "HT", + "HU", + "ID", + "IE", + "IL", + "IM", + "IN", + "IO", + "IQ", + "IR", + "IS", + "IT", + "JE", + "JM", + "JO", + "JP", + "KE", + "KG", + "KH", + "KI", + "KM", + "KN", + "KP", + "KR", + "KW", + "KY", + "KZ", + "LA", + "LB", + "LC", + "LI", + "LK", + "LR", + "LS", + "LT", + "LU", + "LV", + "LY", + "MA", + "MC", + "MD", + "ME", + "MF", + "MG", + "MH", + "MK", + "ML", + "MM", + "MN", + "MO", + "MP", + "MQ", + "MR", + "MS", + "MT", + "MU", + "MV", + "MW", + "MX", + "MY", + "MZ", + "NA", + "NC", + "NE", + "NF", + "NG", + "NI", + "NL", + "NO", + "NP", + "NR", + "NU", + "NZ", + "OM", + "PA", + "PE", + "PF", + "PG", + "PH", + "PK", + "PL", + "PM", + "PN", + "PR", + "PS", + "PT", + "PW", + "PY", + "QA", + "RE", + "RO", + "RS", + "RU", + "RW", + "SA", + "SB", + "SC", + "SD", + "SE", + "SG", + "SH", + "SI", + "SJ", + "SK", + "SL", + "SM", + "SN", + "SO", + "SR", + "SS", + "ST", + "SV", + "SX", + "SY", + "SZ", + "TC", + "TD", + "TF", + "TG", + "TH", + "TJ", + "TK", + "TL", + "TM", + "TN", + "TO", + "TR", + "TT", + "TV", + "TW", + "TZ", + "UA", + "UG", + "UM", + "US", + "UY", + "UZ", + "VA", + "VC", + "VE", + "VG", + "VI", + "VN", + "VU", + "WF", + "WS", + "YE", + "YT", + "ZA", + "ZM", + "ZW" + ], + "type": "string" + }, + "date": { + "$comment": "Note to tool implementers: it is necessary to cast YAML 'date' objects to string objects when validating against this schema.", + "examples": [ + "1900-01-01", + "2020-12-31" + ], + "format": "date", + "pattern": "^[0-9]{4}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string" + }, + "doi": { + "description": "The DOI of the work (i.e., 10.5281/zenodo.1003150, not the resolver URL http://doi.org/10.5281/zenodo.1003150).", + "examples": [ + "10.5281/zenodo.1003150" + ], + "pattern": "^10\\.\\d{4,9}(\\.\\d+)?/[A-Za-z0-9:/_;\\-\\.\\(\\)\\[\\]\\\\]+$", + "type": "string" + }, + "email": { + "description": "An email address.", + "pattern": "^[\\S]+@[\\S]+\\.[\\S]{2,}$", + "type": "string" + }, + "entity": { + "additionalProperties": false, + "description": "An entity, i.e., an institution, team, research group, company, conference, etc., as opposed to a single natural person.", + "properties": { + "address": { + "$ref": "#/definitions/address", + "description": "The entity's address." + }, + "alias": { + "$ref": "#/definitions/alias", + "description": "The entity's alias." + }, + "city": { + "$ref": "#/definitions/city", + "description": "The entity's city." + }, + "country": { + "$ref": "#/definitions/country", + "description": "The entity's country." + }, + "date-end": { + "$ref": "#/definitions/date", + "description": "The entity's ending date, e.g., when the entity is a conference." + }, + "date-start": { + "$ref": "#/definitions/date", + "description": "The entity's starting date, e.g., when the entity is a conference." + }, + "email": { + "$ref": "#/definitions/email", + "description": "The entity's email address." + }, + "fax": { + "$ref": "#/definitions/fax", + "description": "The entity's fax number." + }, + "location": { + "description": "The entity's location, e.g., when the entity is a conference.", + "minLength": 1, + "type": "string" + }, + "name": { + "description": "The entity's name.", + "minLength": 1, + "type": "string" + }, + "orcid": { + "$ref": "#/definitions/orcid", + "description": "The entity's orcid." + }, + "post-code": { + "$ref": "#/definitions/post-code", + "description": "The entity's post code." + }, + "region": { + "$ref": "#/definitions/region", + "description": "The entity's region." + }, + "tel": { + "$ref": "#/definitions/tel", + "description": "The entity's telephone number." + }, + "website": { + "$ref": "#/definitions/url", + "description": "The entity's website." + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "fax": { + "description": "A fax number.", + "minLength": 1, + "type": "string" + }, + "identifier": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/identifier-description" + }, + "type": { + "enum": [ + "doi" + ], + "type": "string" + }, + "value": { + "$ref": "#/definitions/doi" + } + }, + "required": [ + "type", + "value" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/identifier-description" + }, + "type": { + "enum": [ + "url" + ], + "type": "string" + }, + "value": { + "$ref": "#/definitions/url" + } + }, + "required": [ + "type", + "value" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/identifier-description" + }, + "type": { + "enum": [ + "swh" + ], + "type": "string" + }, + "value": { + "$ref": "#/definitions/swh-identifier" + } + }, + "required": [ + "type", + "value" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/identifier-description" + }, + "type": { + "enum": [ + "other" + ], + "type": "string" + }, + "value": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "type": "object" + } + ], + "description": "An identifier for a work." + }, + "identifier-description": { + "description": "A description for a specific identifier value.", + "examples": [ + "The version DOI for this version, which has a relation childOf with the concept DOI specified in the doi field in the root of this file.", + "The identifier provided by Archival Repository, which points to this version of the software." + ], + "minLength": 1, + "type": "string" + }, + "license": { + "description": "An SPDX license identifier.", + "oneOf": [ + { + "$ref": "#/definitions/license-enum", + "examples": [ + "Apache-2.0", + "MIT" + ] + }, + { + "$comment": "When there are multiple licenses, it is assumed their relationship is OR, not AND", + "examples": [ + [ + "Apache-2.0", + "MIT" + ], + [ + "GPL-3.0", + "GPL-3.0-or-later" + ] + ], + "items": { + "$ref": "#/definitions/license-enum" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + } + ] + }, + "license-enum": { + "$comment": "Use https://github.com/citation-file-format/get-spdx-licenses to update this enum in the future", + "description": "SPDX license list; releaseDate=2021-05-14; source=https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json", + "enum": [ + "0BSD", + "AAL", + "Abstyles", + "Adobe-2006", + "Adobe-Glyph", + "ADSL", + "AFL-1.1", + "AFL-1.2", + "AFL-2.0", + "AFL-2.1", + "AFL-3.0", + "Afmparse", + "AGPL-1.0", + "AGPL-1.0-only", + "AGPL-1.0-or-later", + "AGPL-3.0", + "AGPL-3.0-only", + "AGPL-3.0-or-later", + "Aladdin", + "AMDPLPA", + "AML", + "AMPAS", + "ANTLR-PD", + "ANTLR-PD-fallback", + "Apache-1.0", + "Apache-1.1", + "Apache-2.0", + "APAFML", + "APL-1.0", + "APSL-1.0", + "APSL-1.1", + "APSL-1.2", + "APSL-2.0", + "Artistic-1.0", + "Artistic-1.0-cl8", + "Artistic-1.0-Perl", + "Artistic-2.0", + "Bahyph", + "Barr", + "Beerware", + "BitTorrent-1.0", + "BitTorrent-1.1", + "blessing", + "BlueOak-1.0.0", + "Borceux", + "BSD-1-Clause", + "BSD-2-Clause", + "BSD-2-Clause-FreeBSD", + "BSD-2-Clause-NetBSD", + "BSD-2-Clause-Patent", + "BSD-2-Clause-Views", + "BSD-3-Clause", + "BSD-3-Clause-Attribution", + "BSD-3-Clause-Clear", + "BSD-3-Clause-LBNL", + "BSD-3-Clause-Modification", + "BSD-3-Clause-No-Nuclear-License", + "BSD-3-Clause-No-Nuclear-License-2014", + "BSD-3-Clause-No-Nuclear-Warranty", + "BSD-3-Clause-Open-MPI", + "BSD-4-Clause", + "BSD-4-Clause-Shortened", + "BSD-4-Clause-UC", + "BSD-Protection", + "BSD-Source-Code", + "BSL-1.0", + "BUSL-1.1", + "bzip2-1.0.5", + "bzip2-1.0.6", + "C-UDA-1.0", + "CAL-1.0", + "CAL-1.0-Combined-Work-Exception", + "Caldera", + "CATOSL-1.1", + "CC-BY-1.0", + "CC-BY-2.0", + "CC-BY-2.5", + "CC-BY-3.0", + "CC-BY-3.0-AT", + "CC-BY-3.0-US", + "CC-BY-4.0", + "CC-BY-NC-1.0", + "CC-BY-NC-2.0", + "CC-BY-NC-2.5", + "CC-BY-NC-3.0", + "CC-BY-NC-4.0", + "CC-BY-NC-ND-1.0", + "CC-BY-NC-ND-2.0", + "CC-BY-NC-ND-2.5", + "CC-BY-NC-ND-3.0", + "CC-BY-NC-ND-3.0-IGO", + "CC-BY-NC-ND-4.0", + "CC-BY-NC-SA-1.0", + "CC-BY-NC-SA-2.0", + "CC-BY-NC-SA-2.5", + "CC-BY-NC-SA-3.0", + "CC-BY-NC-SA-4.0", + "CC-BY-ND-1.0", + "CC-BY-ND-2.0", + "CC-BY-ND-2.5", + "CC-BY-ND-3.0", + "CC-BY-ND-4.0", + "CC-BY-SA-1.0", + "CC-BY-SA-2.0", + "CC-BY-SA-2.0-UK", + "CC-BY-SA-2.1-JP", + "CC-BY-SA-2.5", + "CC-BY-SA-3.0", + "CC-BY-SA-3.0-AT", + "CC-BY-SA-4.0", + "CC-PDDC", + "CC0-1.0", + "CDDL-1.0", + "CDDL-1.1", + "CDL-1.0", + "CDLA-Permissive-1.0", + "CDLA-Sharing-1.0", + "CECILL-1.0", + "CECILL-1.1", + "CECILL-2.0", + "CECILL-2.1", + "CECILL-B", + "CECILL-C", + "CERN-OHL-1.1", + "CERN-OHL-1.2", + "CERN-OHL-P-2.0", + "CERN-OHL-S-2.0", + "CERN-OHL-W-2.0", + "ClArtistic", + "CNRI-Jython", + "CNRI-Python", + "CNRI-Python-GPL-Compatible", + "Condor-1.1", + "copyleft-next-0.3.0", + "copyleft-next-0.3.1", + "CPAL-1.0", + "CPL-1.0", + "CPOL-1.02", + "Crossword", + "CrystalStacker", + "CUA-OPL-1.0", + "Cube", + "curl", + "D-FSL-1.0", + "diffmark", + "DOC", + "Dotseqn", + "DRL-1.0", + "DSDP", + "dvipdfm", + "ECL-1.0", + "ECL-2.0", + "eCos-2.0", + "EFL-1.0", + "EFL-2.0", + "eGenix", + "Entessa", + "EPICS", + "EPL-1.0", + "EPL-2.0", + "ErlPL-1.1", + "etalab-2.0", + "EUDatagrid", + "EUPL-1.0", + "EUPL-1.1", + "EUPL-1.2", + "Eurosym", + "Fair", + "Frameworx-1.0", + "FreeBSD-DOC", + "FreeImage", + "FSFAP", + "FSFUL", + "FSFULLR", + "FTL", + "GD", + "GFDL-1.1", + "GFDL-1.1-invariants-only", + "GFDL-1.1-invariants-or-later", + "GFDL-1.1-no-invariants-only", + "GFDL-1.1-no-invariants-or-later", + "GFDL-1.1-only", + "GFDL-1.1-or-later", + "GFDL-1.2", + "GFDL-1.2-invariants-only", + "GFDL-1.2-invariants-or-later", + "GFDL-1.2-no-invariants-only", + "GFDL-1.2-no-invariants-or-later", + "GFDL-1.2-only", + "GFDL-1.2-or-later", + "GFDL-1.3", + "GFDL-1.3-invariants-only", + "GFDL-1.3-invariants-or-later", + "GFDL-1.3-no-invariants-only", + "GFDL-1.3-no-invariants-or-later", + "GFDL-1.3-only", + "GFDL-1.3-or-later", + "Giftware", + "GL2PS", + "Glide", + "Glulxe", + "GLWTPL", + "gnuplot", + "GPL-1.0", + "GPL-1.0-only", + "GPL-1.0-or-later", + "GPL-1.0+", + "GPL-2.0", + "GPL-2.0-only", + "GPL-2.0-or-later", + "GPL-2.0-with-autoconf-exception", + "GPL-2.0-with-bison-exception", + "GPL-2.0-with-classpath-exception", + "GPL-2.0-with-font-exception", + "GPL-2.0-with-GCC-exception", + "GPL-2.0+", + "GPL-3.0", + "GPL-3.0-only", + "GPL-3.0-or-later", + "GPL-3.0-with-autoconf-exception", + "GPL-3.0-with-GCC-exception", + "GPL-3.0+", + "gSOAP-1.3b", + "HaskellReport", + "Hippocratic-2.1", + "HPND", + "HPND-sell-variant", + "HTMLTIDY", + "IBM-pibs", + "ICU", + "IJG", + "ImageMagick", + "iMatix", + "Imlib2", + "Info-ZIP", + "Intel", + "Intel-ACPI", + "Interbase-1.0", + "IPA", + "IPL-1.0", + "ISC", + "JasPer-2.0", + "JPNIC", + "JSON", + "LAL-1.2", + "LAL-1.3", + "Latex2e", + "Leptonica", + "LGPL-2.0", + "LGPL-2.0-only", + "LGPL-2.0-or-later", + "LGPL-2.0+", + "LGPL-2.1", + "LGPL-2.1-only", + "LGPL-2.1-or-later", + "LGPL-2.1+", + "LGPL-3.0", + "LGPL-3.0-only", + "LGPL-3.0-or-later", + "LGPL-3.0+", + "LGPLLR", + "Libpng", + "libpng-2.0", + "libselinux-1.0", + "libtiff", + "LiLiQ-P-1.1", + "LiLiQ-R-1.1", + "LiLiQ-Rplus-1.1", + "Linux-OpenIB", + "LPL-1.0", + "LPL-1.02", + "LPPL-1.0", + "LPPL-1.1", + "LPPL-1.2", + "LPPL-1.3a", + "LPPL-1.3c", + "MakeIndex", + "MirOS", + "MIT", + "MIT-0", + "MIT-advertising", + "MIT-CMU", + "MIT-enna", + "MIT-feh", + "MIT-Modern-Variant", + "MIT-open-group", + "MITNFA", + "Motosoto", + "mpich2", + "MPL-1.0", + "MPL-1.1", + "MPL-2.0", + "MPL-2.0-no-copyleft-exception", + "MS-PL", + "MS-RL", + "MTLL", + "MulanPSL-1.0", + "MulanPSL-2.0", + "Multics", + "Mup", + "NAIST-2003", + "NASA-1.3", + "Naumen", + "NBPL-1.0", + "NCGL-UK-2.0", + "NCSA", + "Net-SNMP", + "NetCDF", + "Newsletr", + "NGPL", + "NIST-PD", + "NIST-PD-fallback", + "NLOD-1.0", + "NLPL", + "Nokia", + "NOSL", + "Noweb", + "NPL-1.0", + "NPL-1.1", + "NPOSL-3.0", + "NRL", + "NTP", + "NTP-0", + "Nunit", + "O-UDA-1.0", + "OCCT-PL", + "OCLC-2.0", + "ODbL-1.0", + "ODC-By-1.0", + "OFL-1.0", + "OFL-1.0-no-RFN", + "OFL-1.0-RFN", + "OFL-1.1", + "OFL-1.1-no-RFN", + "OFL-1.1-RFN", + "OGC-1.0", + "OGDL-Taiwan-1.0", + "OGL-Canada-2.0", + "OGL-UK-1.0", + "OGL-UK-2.0", + "OGL-UK-3.0", + "OGTSL", + "OLDAP-1.1", + "OLDAP-1.2", + "OLDAP-1.3", + "OLDAP-1.4", + "OLDAP-2.0", + "OLDAP-2.0.1", + "OLDAP-2.1", + "OLDAP-2.2", + "OLDAP-2.2.1", + "OLDAP-2.2.2", + "OLDAP-2.3", + "OLDAP-2.4", + "OLDAP-2.5", + "OLDAP-2.6", + "OLDAP-2.7", + "OLDAP-2.8", + "OML", + "OpenSSL", + "OPL-1.0", + "OSET-PL-2.1", + "OSL-1.0", + "OSL-1.1", + "OSL-2.0", + "OSL-2.1", + "OSL-3.0", + "Parity-6.0.0", + "Parity-7.0.0", + "PDDL-1.0", + "PHP-3.0", + "PHP-3.01", + "Plexus", + "PolyForm-Noncommercial-1.0.0", + "PolyForm-Small-Business-1.0.0", + "PostgreSQL", + "PSF-2.0", + "psfrag", + "psutils", + "Python-2.0", + "Qhull", + "QPL-1.0", + "Rdisc", + "RHeCos-1.1", + "RPL-1.1", + "RPL-1.5", + "RPSL-1.0", + "RSA-MD", + "RSCPL", + "Ruby", + "SAX-PD", + "Saxpath", + "SCEA", + "Sendmail", + "Sendmail-8.23", + "SGI-B-1.0", + "SGI-B-1.1", + "SGI-B-2.0", + "SHL-0.5", + "SHL-0.51", + "SimPL-2.0", + "SISSL", + "SISSL-1.2", + "Sleepycat", + "SMLNJ", + "SMPPL", + "SNIA", + "Spencer-86", + "Spencer-94", + "Spencer-99", + "SPL-1.0", + "SSH-OpenSSH", + "SSH-short", + "SSPL-1.0", + "StandardML-NJ", + "SugarCRM-1.1.3", + "SWL", + "TAPR-OHL-1.0", + "TCL", + "TCP-wrappers", + "TMate", + "TORQUE-1.1", + "TOSL", + "TU-Berlin-1.0", + "TU-Berlin-2.0", + "UCL-1.0", + "Unicode-DFS-2015", + "Unicode-DFS-2016", + "Unicode-TOU", + "Unlicense", + "UPL-1.0", + "Vim", + "VOSTROM", + "VSL-1.0", + "W3C", + "W3C-19980720", + "W3C-20150513", + "Watcom-1.0", + "Wsuipa", + "WTFPL", + "wxWindows", + "X11", + "Xerox", + "XFree86-1.1", + "xinetd", + "Xnet", + "xpp", + "XSkat", + "YPL-1.0", + "YPL-1.1", + "Zed", + "Zend-2.0", + "Zimbra-1.3", + "Zimbra-1.4", + "Zlib", + "zlib-acknowledgement", + "ZPL-1.1", + "ZPL-2.0", + "ZPL-2.1" + ], + "type": "string" + }, + "orcid": { + "description": "Identifier for an author, see https://orcid.org.", + "format": "uri", + "pattern": "https://orcid\\.org/[0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9]{3}[0-9X]{1}", + "type": "string" + }, + "person": { + "additionalProperties": false, + "description": "A person.", + "properties": { + "address": { + "$ref": "#/definitions/address", + "description": "The person's address." + }, + "affiliation": { + "description": "The person's affilitation.", + "minLength": 1, + "type": "string" + }, + "alias": { + "$ref": "#/definitions/alias", + "description": "The person's alias." + }, + "city": { + "$ref": "#/definitions/city", + "description": "The person's city." + }, + "country": { + "$ref": "#/definitions/country", + "description": "The person's country." + }, + "email": { + "$ref": "#/definitions/email", + "description": "The person's email address." + }, + "family-names": { + "description": "The person's family names.", + "minLength": 1, + "type": "string" + }, + "fax": { + "$ref": "#/definitions/fax", + "description": "The person's fax number." + }, + "given-names": { + "description": "The person's given names.", + "minLength": 1, + "type": "string" + }, + "name-particle": { + "description": "The person's name particle, e.g., a nobiliary particle or a preposition meaning 'of' or 'from' (for example 'von' in 'Alexander von Humboldt').", + "examples": [ + "von" + ], + "minLength": 1, + "type": "string" + }, + "name-suffix": { + "description": "The person's name-suffix, e.g. 'Jr.' for Sammy Davis Jr. or 'III' for Frank Edwin Wright III.", + "examples": [ + "Jr.", + "III" + ], + "minLength": 1, + "type": "string" + }, + "orcid": { + "$ref": "#/definitions/orcid", + "description": "The person's ORCID." + }, + "post-code": { + "$ref": "#/definitions/post-code", + "description": "The person's post-code." + }, + "region": { + "$ref": "#/definitions/region", + "description": "The person's region." + }, + "tel": { + "$ref": "#/definitions/tel", + "description": "The person's phone number." + }, + "website": { + "$ref": "#/definitions/url", + "description": "The person's website." + } + }, + "type": "object" + }, + "post-code": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "number" + } + ], + "description": "A post code." + }, + "reference": { + "additionalProperties": false, + "description": "A reference to a work.", + "properties": { + "abbreviation": { + "description": "The abbreviation of a work.", + "minLength": 1, + "type": "string" + }, + "abstract": { + "description": "The abstract of a work.", + "minLength": 1, + "type": "string" + }, + "authors": { + "description": "The author(s) of a work.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/person" + }, + { + "$ref": "#/definitions/entity" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "collection-doi": { + "$ref": "#/definitions/doi", + "description": "The DOI of a collection containing the work." + }, + "collection-title": { + "description": "The title of a collection or proceedings.", + "minLength": 1, + "type": "string" + }, + "collection-type": { + "description": "The type of a collection.", + "minLength": 1, + "type": "string" + }, + "commit": { + "$ref": "#/definitions/commit" + }, + "conference": { + "$ref": "#/definitions/entity", + "description": "The conference where the work was presented." + }, + "contact": { + "description": "The contact person, group, company, etc. for a work.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/person" + }, + { + "$ref": "#/definitions/entity" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "copyright": { + "description": "The copyright information pertaining to the work.", + "minLength": 1, + "type": "string" + }, + "data-type": { + "description": "The data type of a data set.", + "minLength": 1, + "type": "string" + }, + "database": { + "description": "The name of the database where a work was accessed/is stored.", + "minLength": 1, + "type": "string" + }, + "database-provider": { + "$ref": "#/definitions/entity", + "description": "The provider of the database where a work was accessed/is stored." + }, + "date-accessed": { + "$ref": "#/definitions/date", + "description": "The date the work was accessed." + }, + "date-downloaded": { + "$ref": "#/definitions/date", + "description": "The date the work has been downloaded." + }, + "date-published": { + "$ref": "#/definitions/date", + "description": "The date the work has been published." + }, + "date-released": { + "$ref": "#/definitions/date", + "description": "The date the work has been released." + }, + "department": { + "description": "The department where a work has been produced.", + "minLength": 1, + "type": "string" + }, + "doi": { + "$ref": "#/definitions/doi", + "description": "The DOI of the work." + }, + "edition": { + "description": "The edition of the work.", + "minLength": 1, + "type": "string" + }, + "editors": { + "description": "The editor(s) of a work.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/person" + }, + { + "$ref": "#/definitions/entity" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "editors-series": { + "description": "The editor(s) of a series in which a work has been published.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/person" + }, + { + "$ref": "#/definitions/entity" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "end": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The end page of the work." + }, + "entry": { + "description": "An entry in the collection that constitutes the work.", + "minLength": 1, + "type": "string" + }, + "filename": { + "description": "The name of the electronic file containing the work.", + "minLength": 1, + "type": "string" + }, + "format": { + "description": "The format in which a work is represented.", + "minLength": 1, + "type": "string" + }, + "identifiers": { + "description": "The identifier(s) of the work.", + "items": { + "$ref": "#/definitions/identifier" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "institution": { + "$ref": "#/definitions/entity", + "description": "The institution where a work has been produced or published." + }, + "isbn": { + "description": "The ISBN of the work.", + "pattern": "^[0-9\\- ]{10,17}X?$", + "type": "string" + }, + "issn": { + "description": "The ISSN of the work.", + "pattern": "^\\d{4}-\\d{3}[\\dxX]$", + "type": "string" + }, + "issue": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "number" + } + ], + "description": "The issue of a periodical in which a work appeared." + }, + "issue-date": { + "description": "The publication date of the issue of a periodical in which a work appeared.", + "minLength": 1, + "type": "string" + }, + "issue-title": { + "description": "The name of the issue of a periodical in which the work appeared.", + "minLength": 1, + "type": "string" + }, + "journal": { + "description": "The name of the journal/magazine/newspaper/periodical where the work was published.", + "minLength": 1, + "type": "string" + }, + "keywords": { + "description": "Keywords pertaining to the work.", + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "languages": { + "description": "The language identifier(s) of the work according to ISO 639 language strings.", + "items": { + "maxLength": 3, + "minLength": 2, + "pattern": "^[a-z]{2,3}$", + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "license": { + "$ref": "#/definitions/license" + }, + "license-url": { + "$ref": "#/definitions/url", + "description": "The URL of the license text under which the work is licensed (only for non-standard licenses not included in the SPDX License List)." + }, + "loc-end": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The line of code in the file where the work ends." + }, + "loc-start": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The line of code in the file where the work starts." + }, + "location": { + "$ref": "#/definitions/entity", + "description": "The location of the work." + }, + "medium": { + "description": "The medium of the work.", + "minLength": 1, + "type": "string" + }, + "month": { + "anyOf": [ + { + "maximum": 12, + "minimum": 1, + "type": "integer" + }, + { + "enum": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12" + ], + "type": "string" + } + ], + "description": "The month in which a work has been published." + }, + "nihmsid": { + "description": "The NIHMSID of a work.", + "minLength": 1, + "type": "string" + }, + "notes": { + "description": "Notes pertaining to the work.", + "minLength": 1, + "type": "string" + }, + "number": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "number" + } + ], + "description": "The accession number for a work." + }, + "number-volumes": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The number of volumes making up the collection in which the work has been published." + }, + "pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The number of pages of the work." + }, + "patent-states": { + "description": "The states for which a patent is granted.", + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "pmcid": { + "description": "The PMCID of a work.", + "pattern": "^PMC[0-9]{7}$", + "type": "string" + }, + "publisher": { + "$ref": "#/definitions/entity", + "description": "The publisher who has published the work." + }, + "recipients": { + "description": "The recipient(s) of a personal communication.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/entity" + }, + { + "$ref": "#/definitions/person" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "repository": { + "$ref": "#/definitions/url", + "description": "The URL of the work in a repository (when the repository is neither a source code repository nor a build artifact repository)." + }, + "repository-artifact": { + "$ref": "#/definitions/url", + "description": "The URL of the work in a build artifact/binary repository." + }, + "repository-code": { + "$ref": "#/definitions/url", + "description": "The URL of the work in a source code repository." + }, + "scope": { + "description": "The scope of the reference, e.g., the section of the work it adheres to.", + "minLength": 1, + "type": "string" + }, + "section": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "number" + } + ], + "description": "The section of a work that is referenced." + }, + "senders": { + "description": "The sender(s) of a personal communication.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/entity" + }, + { + "$ref": "#/definitions/person" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "start": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The start page of the work." + }, + "status": { + "description": "The publication status of the work.", + "enum": [ + "abstract", + "advance-online", + "in-preparation", + "in-press", + "preprint", + "submitted" + ], + "type": "string" + }, + "term": { + "description": "The term being referenced if the work is a dictionary or encyclopedia.", + "minLength": 1, + "type": "string" + }, + "thesis-type": { + "description": "The type of the thesis that is the work.", + "minLength": 1, + "type": "string" + }, + "title": { + "description": "The title of the work.", + "minLength": 1, + "type": "string" + }, + "translators": { + "description": "The translator(s) of a work.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/entity" + }, + { + "$ref": "#/definitions/person" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "type": { + "description": "The type of the work.", + "enum": [ + "art", + "article", + "audiovisual", + "bill", + "blog", + "book", + "catalogue", + "conference-paper", + "conference", + "data", + "database", + "dictionary", + "edited-work", + "encyclopedia", + "film-broadcast", + "generic", + "government-document", + "grant", + "hearing", + "historical-work", + "legal-case", + "legal-rule", + "magazine-article", + "manual", + "map", + "multimedia", + "music", + "newspaper-article", + "pamphlet", + "patent", + "personal-communication", + "proceedings", + "report", + "serial", + "slides", + "software-code", + "software-container", + "software-executable", + "software-virtual-machine", + "software", + "sound-recording", + "standard", + "statute", + "thesis", + "unpublished", + "video", + "website" + ], + "type": "string" + }, + "url": { + "$ref": "#/definitions/url", + "description": "The URL of the work." + }, + "version": { + "$ref": "#/definitions/version", + "description": "The version of the work." + }, + "volume": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The volume of the periodical in which a work appeared." + }, + "volume-title": { + "description": "The title of the volume in which the work appeared.", + "minLength": 1, + "type": "string" + }, + "year": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The year in which a work has been published." + }, + "year-original": { + "anyOf": [ + { + "type": "integer" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The year of the original publication." + } + }, + "required": [ + "authors", + "title", + "type" + ], + "type": "object" + }, + "region": { + "description": "A region.", + "minLength": 1, + "type": "string" + }, + "swh-identifier": { + "$comment": "Software Heritage identifiers are documented here: https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html.", + "description": "The Software Heritage identifier (without further qualifiers such as origin, visit, anchor, path).", + "examples": [ + "swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2", + "swh:1:dir:d198bc9d7a6bcf6db04f476d29314f157507d505", + "swh:1:rev:309cf2674ee7a0749978cf8265ab91a60aea0f7d", + "swh:1:rel:22ece559cc7cc2364edc5e5593d63ae8bd229f9f", + "swh:1:snp:c7c108084bc0bf3d81436bf980b46e98bd338453" + ], + "pattern": "^swh:1:(snp|rel|rev|dir|cnt):[0-9a-fA-F]{40}$", + "type": "string" + }, + "tel": { + "description": "A phone number.", + "minLength": 1, + "type": "string" + }, + "url": { + "format": "uri", + "pattern": "^(https|http|ftp|sftp)://.+", + "type": "string" + }, + "version": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "number" + } + ] + } + }, + "description": "A file with citation metadata for software or datasets.", + "properties": { + "abstract": { + "description": "A description of the software or dataset.", + "minLength": 1, + "type": "string" + }, + "authors": { + "description": "The author(s) of the software or dataset.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/person" + }, + { + "$ref": "#/definitions/entity" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "cff-version": { + "description": "The version of CFF used for providing the citation metadata.", + "examples": [ + "1.2.0" + ], + "pattern": "^1\\.2\\.0$", + "type": "string" + }, + "commit": { + "$ref": "#/definitions/commit" + }, + "contact": { + "description": "The contact person, group, company, etc. for the software or dataset.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/person" + }, + { + "$ref": "#/definitions/entity" + } + ] + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "date-released": { + "$ref": "#/definitions/date", + "description": "The date the work has been released." + }, + "doi": { + "$ref": "#/definitions/doi" + }, + "identifiers": { + "description": "The identifiers of the software or dataset.", + "items": { + "$ref": "#/definitions/identifier" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "keywords": { + "description": "Keywords that describe the work.", + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "license": { + "$ref": "#/definitions/license" + }, + "license-url": { + "$ref": "#/definitions/url", + "description": "The URL of the license text under which the software or dataset is licensed (only for non-standard licenses not included in the SPDX License List)." + }, + "message": { + "default": "If you use this software, please cite it using the metadata from this file.", + "description": "A message to the human reader of the file to let them know what to do with the citation metadata.", + "examples": [ + "If you use this software, please cite it using the metadata from this file.", + "Please cite this software using these metadata.", + "Please cite this software using the metadata from 'preferred-citation'." + ], + "minLength": 1, + "type": "string" + }, + "preferred-citation": { + "$ref": "#/definitions/reference", + "description": "A reference to another work that should be cited instead of the software or dataset itself." + }, + "references": { + "description": "Reference(s) to other creative works.", + "items": { + "$ref": "#/definitions/reference" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "repository": { + "$ref": "#/definitions/url", + "description": "The URL of the software or dataset in a repository (when the repository is neither a source code repository nor a build artifact repository).", + "examples": [ + "https://edoc.hu-berlin.de/handle/18452/23016", + "https://ascl.net/2105.013" + ] + }, + "repository-artifact": { + "$ref": "#/definitions/url", + "description": "The URL of the software in a build artifact/binary repository." + }, + "repository-code": { + "$ref": "#/definitions/url", + "description": "The URL of the software or dataset in a source code repository." + }, + "title": { + "description": "The name of the software or dataset.", + "minLength": 1, + "type": "string" + }, + "type": { + "default": "software", + "description": "The type of the work.", + "enum": [ + "dataset", + "software" + ], + "type": "string" + }, + "url": { + "$ref": "#/definitions/url", + "description": "The URL of a landing page/website for the software or dataset." + }, + "version": { + "$ref": "#/definitions/version", + "description": "The version of the software or dataset." + } + }, + "required": [ + "authors", + "cff-version", + "message", + "title" + ], + "title": "Citation File Format", + "type": "object" +} diff --git a/.github/citation-schema.LICENSE b/.github/citation-schema.LICENSE new file mode 100644 index 0000000..e69451d --- /dev/null +++ b/.github/citation-schema.LICENSE @@ -0,0 +1,395 @@ +Creative Commons Attribution 4.0 International Public License + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..d73280c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,11 @@ +## Change + +Describe the problem and resulting behavior. + +## Validation + +List the checks you ran, environment, and anything not tested. + +## Reproducibility impact + +Identify any intentional changes to inputs, scoring, dependencies, CLI behavior or outputs. diff --git a/.github/workflows/cpu.yml b/.github/workflows/cpu.yml new file mode 100644 index 0000000..83b0c70 --- /dev/null +++ b/.github/workflows/cpu.yml @@ -0,0 +1,73 @@ +name: CPU checks + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + cpu: + name: Python ${{ matrix.python }} CPU + runs-on: ubuntu-22.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.11'] + env: + MPLBACKEND: Agg + PYTHONDONTWRITEBYTECODE: '1' + TOKENIZERS_PARALLELISM: 'false' + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: pyproject.toml + - name: Install CPU runtime and development dependencies + run: | + python -m pip install 'torch==2.5.1' --index-url https://download.pytorch.org/whl/cpu + python -m pip install -e '.[benchmark,design,dev,notebook]' + python -m pip check + - name: Regression tests + run: | + python -m pytest -q tests --junitxml="$RUNNER_TEMP/tests.xml" + python - <<'PYTEST' + import os, xml.etree.ElementTree as ET + report = ET.parse(os.path.join(os.environ['RUNNER_TEMP'], 'tests.xml')) + skipped = sum(int(s.get('skipped', 0)) for s in report.getroot().iter('testsuite')) + assert skipped == 0, f'{skipped} tests skipped; required CPU dependencies must be installed' + PYTEST + - name: Documentation, citation and frozen inputs + run: python scripts/check_repository.py + - name: Execute CPU example from a full source archive + run: | + mkdir -p "$RUNNER_TEMP/source-archive" + git archive --format=tar --prefix=EVA/ HEAD | tar -xf - -C "$RUNNER_TEMP/source-archive" + cd "$RUNNER_TEMP/source-archive/EVA" + python scripts/reproduce_historical_benchmark.py --artifact-only --output "$RUNNER_TEMP/cpu-example" + - name: Build wheel and verify installation outside the checkout + run: | + python -m build + python -m pip uninstall -y eva-rna + python -m pip install --no-deps dist/*.whl + python -m pip check + cd "$RUNNER_TEMP" + python -I "$GITHUB_WORKSPACE/scripts/check_installed_package.py" --source-root "$GITHUB_WORKSPACE" + - name: Save test and CPU example reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: cpu-python-${{ matrix.python }} + path: | + ${{ runner.temp }}/tests.xml + ${{ runner.temp }}/cpu-example/report.json + if-no-files-found: ignore diff --git a/.github/workflows/resources.yml b/.github/workflows/resources.yml new file mode 100644 index 0000000..cefd162 --- /dev/null +++ b/.github/workflows/resources.yml @@ -0,0 +1,42 @@ +name: External resources + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - 'scripts/check_resource_links.py' + - 'reproduction/release/resource_links.json' + - '.github/workflows/resources.yml' + +permissions: + contents: read + +jobs: + accessibility: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Record availability independently of CPU regression status + run: python scripts/check_resource_links.py --output resource-access.json + - name: Show observed access failures + run: | + python - <<'PYREPORT' + import json, os + report = json.load(open('resource-access.json')) + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as summary: + summary.write(f"## Resource access\n\n{report['available']}/{report['total']} endpoints available. This is an access report, not model validation.\n\n") + for item in report['checks']: + if item['access'] != 'AVAILABLE': + summary.write(f"- {item['id']}: {item['access']} ({item.get('http_status', item.get('error', 'unknown'))})\n") + PYREPORT + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: external-resource-access + path: resource-access.json diff --git a/.gitignore b/.gitignore index 0af2d2f..228f99a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ venv/ .ruff_cache/ .ipynb_checkpoints/ .DS_Store +results/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 92dc09e..46de3cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,18 @@ -# 1.2.0rc1 Reproducibility candidate +# Changelog + +## Unreleased + +- Organize task guides, resource information and reviewer evidence around a concise README. +- Separate successful Milena execution from optional strict reference comparison. +- Add CPU regression, packaging, documentation and citation checks, with separate external-resource reporting. + +## 1.2.0rc1 Reproducibility candidate - Integrate training/scoring robustness fixes and benchmark arithmetic tests. - Add a pinned public EVA-1.4B Milena inference-to-metric entry point. - Add a single reproduction guide and explicit paper workflow coverage. - Release candidate only; see docs/RELEASE_STATUS.md for validation limitations. -# Changelog - All notable changes to EVA are documented in this file. The project uses Git tags for code releases. A reproducible release should also record the corresponding Hugging Face model revision, dataset revision, and Docker image or build command. diff --git a/CITATION.cff b/CITATION.cff index 5dba55c..74fc17c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,5 +1,5 @@ cff-version: 1.2.0 -message: "Cite the exact software version and commit used. This candidate has no archival DOI yet." +message: "Cite the exact software version and commit used. The main-branch source has no archival code DOI yet." type: software title: "EVA: A Generative Foundation Model for Universal RNA Modeling and Design" authors: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..19be166 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,45 @@ +# Contributing to EVA + +Open an issue before proposing a substantial architecture or scoring change. +Small documentation corrections and bug fixes can be submitted as pull requests. +Keep changes focused and explain the behavior they fix. + +## Development setup + +Use Python 3.10 or 3.11 in a virtual environment from the repository root: + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install -e '.[benchmark,design,dev,notebook]' +python -m pip check +python -m pytest -q tests +python scripts/check_repository.py +``` + +These are CPU checks. Full MoE inference requires the +[GPU environment](docs/INSTALLATION.md). State the tests and environment used +in your pull request, and identify checks that were not run. + +## Reporting a problem + +Include the Git commit, Python/PyTorch/CUDA versions, OS, GPU model if relevant, +installation method, exact command and error log. Prefer a small synthetic or +openly available non-sensitive input. Remove credentials and confidential data. + +## Reproducibility changes + +Preserve IDs, checksums and scoring conventions. Do not fix a failure by +silently dropping samples, changing labels, substituting a model or changing +the objective. Describe deliberate API/protocol changes and test the affected +behavior. Historical snapshots and expected-result records retain their hashes; +add dated evidence instead of rewriting measurements. Do not commit checkpoints, +generated results or local environments. + +## Package validation + +Run `python -m build`, then test the built wheel outside the checkout as in CI. +The wheel contains `eva`, `tools` and package data, not the full training tree +or paper inputs. Use the complete source archive for paper reproduction. +External resource checks report access problems separately from CPU tests; +unavailable historical weights are not silently replaced with current releases. diff --git a/README.md b/README.md index c2dc825..4c9d97f 100644 --- a/README.md +++ b/README.md @@ -1,887 +1,218 @@ # EVA: A Long-Context Generative Foundation Model for Versatile RNA Design -For the reproducible environment, full-assay scoring command, workflow coverage, -and known missing historical artifacts, start with [Reproduction](docs/REPRODUCTION.md). -Running released code and matching a paper result are separate validation steps. - -
- OpenRNA -
-
- -[![Paper](https://img.shields.io/badge/Paper-Paper-blue)](https://www.biorxiv.org/content/10.64898/2026.03.17.712398v1) -[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-EVA-ffc107)](https://huggingface.co/GENTEL-Lab/EVA) -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) -[![Website](https://img.shields.io/badge/Website-EVA-green)](https://evabio.net/) - + EVA
-
🧬 A long-context generative foundation model for universal RNA modeling and design.
+[Paper](https://www.biorxiv.org/content/10.64898/2026.03.17.712398v1) · +[Model weights](https://huggingface.co/GENTEL-Lab/EVA) · +[OpenRNA data](https://huggingface.co/datasets/GENTEL-Lab/OpenRNA-v1-114M) · +[Website](https://evabio.net/) · [Apache-2.0 license](LICENSE) -
- -## Reproducing the paper - -Start with **[the reproduction guide](docs/REPRODUCTION.md)** for installation, -a pinned EVA-1.4B checkpoint, the complete 135-sequence Milena workflow, and -separate commands for recalculating stored predictions and plotting summaries. -See [paper workflow coverage](docs/PAPER_WORKFLOWS.md) for model resources and -components that still require paper-specific artifacts. This revision is a -release candidate; current validation and release status are in -[release status](docs/RELEASE_STATUS.md). - -**EVA** (Evolutionary Versatile Architect) is a generative RNA foundation model trained on **OpenRNA v1**, a curated atlas of 114 million full-length RNA sequences spanning all domains of life. Built on a 1.4B-parameter decoder-only Transformer with a Mixture-of-Experts (MoE) backbone and an 8,192-token context window, EVA unifies RNA sequence scoring and controllable design within a single framework. - - -## Why Use EVA? - -You should consider EVA for the reasons as follows: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
🔓Released Code and AssetsPublic source, checkpoints and datasets support inference and selected reproduction workflows. Historical configurations, dense-model code and per-experiment checkpoint linkage are not all established; see the reproduction guide.
📏8x Larger Context WindowAn 8,192-token context window supports longer RNA inputs. Sequences and conditioning tokens must fit within this limit; overlength inputs require an explicitly documented processing protocol.
🗄️7x More Training DataTrained on OpenRNA v1 — 114M full-length RNA sequences across all domains of life (Eukaryota, Bacteria, Archaea, Viruses), 7.7x larger than the RNAcentral v25.0
🏆SOTA Fitness PredictionAchieves state-of-the-art zero-shot fitness prediction on ncRNA and mRNA, no task-specific fine-tuning needed
🎯Conditional RNA GenerationCLM generation and GLM span infilling support RNA design. Performance claims depend on the specific assays, metrics and controls reported in the manuscript.
🧬11 RNA Types SupportedControllable generation across 11 RNA classes (mRNA, tRNA, rRNA, lncRNA, snRNA, snoRNA, miRNA, and more) conditioned on RNA type and taxonomic lineage
⚙️Novel Architecture, Capabilities & Training1.4B-parameter MoE decoder-only Transformer with dual generation paradigms — CLM for de novo design & GLM for domain redesign — unified in a single model
- -
- - -## Key Modules - -| Module | Path | Description | -|--------|------|-------------| -| Model Architecture | `eva/` | MoE Transformer backbone (modeling, attention, MoE routing, causal LM head) | -| Lineage Tokenizer | `eva/lineage_tokenizer.py` | Tokenizer for taxonomic lineage strings | -| Generation CLI | `tools/generate.py` | Entry point for CLM and GLM sequence generation | -| Scoring CLI | `tools/predict.py` | Entry point for log-likelihood scoring | -| Directed Evolution | `tools/directed_evolution.py` | In-silico directed evolution pipeline | -| Generators | `tools/utils/generators/` | CLM autoregressive and GLM span-infilling implementations | -| Scorers | `tools/utils/scorers/` | Sequence scoring logic and batch workers | -| Condition Control | `tools/utils/conditions/` | RNA type and taxonomic lineage conditioning | -| Model Loader & Sampler | `tools/utils/model/` | Checkpoint loading and sampling strategies | -| Config Templates | `config/` | YAML configuration examples for generation, scoring, and directed evolution | - -
- -### Our Journey with EVA Starts Here 👋 - -> Click below to expand the **Table of Contents** and explore each section in detail. - -
-Table of Contents - -
- -- [Quick Start](#quick-start) -- [Installation](#installation) -- [Release and Versioning](#release-and-versioning) -- [Repository Paths](#repository-paths) -- [Condition Control](#condition-control) - - [RNA Types](#rna-types) - - [Species/Lineage](#specieslineage) -- [Generation](#generation) - - [CLM](#clm) - - [Unconditional Generation](#unconditional-generation) - - [Conditional Generation](#conditional-generation) - - [Continuation Mode](#continuation-mode) - - [GLM](#glm) - - [Unconditional Infilling](#unconditional-infilling) - - [Conditional Infilling](#conditional-infilling) - - [Span Parameters](#span-parameters) - - [Sampling Parameters](#sampling-parameters) -- [Scoring](#scoring) - - [RNA Mode](#rna-mode) - - [Protein Mode](#protein-mode) -- [Directed Evolution](#directed-evolution) - - [Usage](#usage) - - [Key Parameters](#key-parameters) -- [Batch Processing with YAML](#batch-processing-with-yaml) - - [Generation Config Example](#generation-config-example) - - [Scoring Config Example](#scoring-config-example) - - [Running](#running) -- [Input/Output Formats](#inputoutput-formats) -- [Data Availability](#data-availability) - - [Model & Environment](#model--environment) - - [Experiment & Benchmark Data](#experiment--benchmark-data) - - [Auxiliary Data](#auxiliary-data) -- [Citation](#citation) -- [License](#license) - -
- -
- -
+EVA is a generative RNA foundation model for sequence scoring and conditional +sequence modeling. The released model family includes a 1.4B-parameter +mixture-of-experts model with an 8,192-token context window, trained on OpenRNA v1. +This repository contains model code, command-line tools and paper reproduction +workflows. Model weights and large datasets are downloaded separately. +## Start here +| I want to… | Start with | Requirements | +|---|---|---| +| Check a bundled result without a GPU | [CPU result recalculation](docs/REPRODUCTION.md#cpu-recalculate-stored-predictions) | Python 3.10/3.11 only | +| Run a benchmark from model weights | [135-sequence Milena example](docs/REPRODUCTION.md#gpu-run-the-complete-milena-benchmark) | Linux/NVIDIA GPU; Docker recommended | +| Use scoring or generation | [CLI guide](docs/USAGE.md) | Compatible GPU runtime and a checkpoint | +| Train or fine-tune | [Training workflow](docs/REPRODUCTION.md#training-and-fine-tuning) | Complete source checkout and GPU runtime | +| Find a paper result or comparison model | [Paper workflows](docs/PAPER_WORKFLOWS.md) | Requirements vary by workflow | ## Quick Start -EVA is distributed as source code plus separately downloaded model weights and datasets. The Python package provides importable modules and command-line entry points; large checkpoints remain outside the package. - -Run this quick start inside the compatible GPU environment in -[Installation](#installation). On a clean machine, first build/start Option B's -Docker container; the Python install alone does not supply CUDA extensions. +Clone the repository and recalculate the bundled Milena benchmark metric: ```bash git clone https://github.com/GENTEL-Lab/EVA.git cd EVA -python3 -m pip install -e '.[benchmark,design]' - -python3 -c "import eva; print(eva.__version__)" -eva-generate --help -eva-predict --help -eva-evolve --help +python3 scripts/reproduce_historical_benchmark.py \ + --artifact-only --output results/milena_stored ``` -The commands above install Python entry points, not the compiled GPU extensions. -Use the Docker environment below for GPU inference, or install the exact compiled -dependencies from `docker/Dockerfile` into a matching PyTorch/CUDA environment. -`--help` alone is not an inference test. - -Download a pinned checkpoint before running model inference: - -```bash -python scripts/download_reproduction_checkpoint.py --model EVA_21M --destination ./checkpoint -``` - -Generate a small batch of human mRNA sequences: - -```bash -eva-generate \ - --checkpoint ./checkpoint/EVA_21M \ - --format clm \ - --rna_type mRNA \ - --taxid 9606 \ - --num_seqs 10 \ - --output ./data/output/demo.fa -``` - -The legacy script entry points are still supported for source-tree workflows, for example `python tools/generate.py --help`. - -
+This standard-library-only example writes `report.json` and +`sequence_label_audit.csv` for all 135 stored predictions. The expected Spearman +correlation is **0.8360456283218484**, and the command returns 0. It recalculates +an archived result; the [GPU example](docs/REPRODUCTION.md#gpu-run-the-complete-milena-benchmark) +performs new model inference. Use a new output directory when rerunning. ## Installation -Prerequisites: - -- Python 3.10 or 3.11 for local source installs. -- CUDA-capable GPU runtime for full MoE inference. -- Docker is recommended when reproducing the released GPU environment. - -### Option A: Local source install - -Use this path when you want to import EVA modules or run the command-line tools from a cloned repository: - -```bash -git clone https://github.com/GENTEL-Lab/EVA.git -cd EVA -python3 -m pip install -e . -``` - -For documentation and release checks: - -```bash -python3 -m pip install -e ".[dev]" -``` - -For notebook-only figure workflows: - -```bash -python3 -m pip install -e ".[notebook]" -``` - -The default package dependencies keep installation lightweight enough for code reuse. Full 1.4B MoE inference requires the CUDA stack used by the released Docker image, including GPU-specific packages such as `megablocks` and `grouped_gemm`. - -### Option B: Docker runtime - -Build the image from the repository root: - -```bash -docker build -f docker/Dockerfile -t eva:latest . -``` - -The image is based on `pytorch/pytorch:2.5.1-cuda12.4-cudnn9-devel` and compiles -`grouped-gemm`, `stanford-stk` and `megablocks` from source -against the preinstalled PyTorch 2.5.1 (none of them publish usable prebuilt -wheels). The first build takes roughly 30-60 minutes; subsequent builds reuse -the Docker layer cache. `TORCH_CUDA_ARCH_LIST` is fixed to `7.5;8.0;9.0` -(Tesla T4, Ampere/Ada via binary compatibility, Hopper). - -Run an interactive GPU container with checkpoint and output folders mounted from the host: - -```bash -mkdir -p checkpoint data/output -docker run --gpus device=0 --name eva-repro --rm -it \ - -v "$PWD":/eva \ - -v "$PWD/checkpoint":/eva/checkpoint \ - -v "$PWD/data/output":/eva/data/output \ - eva:latest bash -``` - -Inside the container, use either the installed entry points: - -```bash -eva-generate --help -eva-predict --help -``` - -or the source-tree scripts: - -```bash -python /eva/tools/generate.py --help -python /eva/tools/predict.py --help -``` - -The image entrypoint (`docker/entrypoint.sh`, backed by `eva/_runtime_env.py`) -verifies the Triton CUDA driver environment at container start. It only -intervenes when Triton's default discovery is broken (e.g. Singularity/Apptainer -images with driver-library placeholders) and then exports -`TRITON_LIBCUDA_PATH` plus a writable `TRITON_CACHE_DIR`. The same check runs -on every `import eva`, which covers launch paths where container entrypoints -are not executed. - -### Option C: Singularity / Apptainer (HPC clusters) - -HPC users usually cannot run Docker. Convert the image with the definition -file on any machine that has Docker: - -```bash -docker build -f docker/Dockerfile -t eva:latest . -singularity build eva_latest.sif docker/EVA.def -``` - -When starting from a released docker-archive tarball (for example -`eva_latest.tar.gz` from Zenodo): - -```bash -docker load -i eva_latest.tar.gz -singularity build eva_latest.sif docker/EVA.def -``` - -Always launch with `--nv` so the host NVIDIA driver is available: - -```bash -singularity shell --nv eva_latest.sif -singularity exec --nv --bind /path/to/checkpoint:/eva/checkpoint \ - eva_latest.sif python /eva/tools/generate.py --help -``` - -Verify the runtime before running notebooks or training: - -```bash -singularity exec --cleanenv --nv eva_latest.sif \ - python -c "from triton.runtime import driver; print(driver.active)" -``` - -### Troubleshooting: Triton `undefined symbol: cuModuleGetFunction` - -Triton 3.1.0 compiles `cuda_utils.so` on first use and links it against -`-lcuda` at link time, resolving through the ldconfig cache rather than -`LD_LIBRARY_PATH`. Images built via `docker commit` on GPU hosts contain -0-byte `libcuda.so.*` placeholders: Docker `--gpus` mounts the real driver -over them (masking the bug), while Singularity `--nv` leaves them visible, -so the linker binds against an empty file and every Triton import fails -with `undefined symbol: cuModuleGetFunction`. - -Images built from the current repository fix this at the source: the -placeholders are removed at build time, and `eva/_runtime_env.py` points -Triton at the injected driver via `TRITON_LIBCUDA_PATH` on every -`import eva`. Users of images released before this fix must rebuild from -the current repository. - -Verify after any launch method: - -```bash -python -c "from triton.runtime import driver; print(driver.active)" -``` - -### Pre-release smoke test - -Both runtimes must pass before an image is published; Docker alone hides -driver-placeholder bugs that only surface under Singularity `--nv`: - -```bash -docker/smoke_test.sh docker eva:latest -docker/smoke_test.sh singularity eva_latest.sif -``` - -### Model download - -For reproducible runs, prefer a fixed Hugging Face revision: - -```bash -export HF_MODEL_REVISION=514db6705637c1ec963b728768fc9b34728699ee -huggingface-cli download GENTEL-Lab/EVA \ - --revision "$HF_MODEL_REVISION" \ - --local-dir ./checkpoint -``` - -If no revision is specified, Hugging Face downloads the current default branch of the model repository. - -
- -## Release and Versioning - -Use versioned releases for reproducible downstream use. A release should bind together: - -- a Git tag for this repository, for example `v1.1.1`; -- the Python package version from `eva.__version__`; -- the matching `CHANGELOG.md` entry; -- the Hugging Face model revision used for inference; -- the dataset revision used for full reproduction; -- the Docker image tag or the exact Docker build command. - -Recommended release workflow: - -```bash -git checkout v1.1.1 -python3 -m pip install -e . -python3 -c "import eva; print(eva.__version__)" -eva-generate --help -``` - -You can also install directly from a release tag: - -```bash -python3 -m pip install "git+https://github.com/GENTEL-Lab/EVA.git@v1.1.1" -``` - -The repository is package-ready through `pyproject.toml`, but release artifacts such as `dist/`, wheels, and source tarballs should be generated during release and not committed to the repository. PyPI or GitHub Packages publication is optional and should only be enabled when the supported dependency matrix is stable. - -
- -## Repository Paths - -The repository keeps code, examples, and large runtime assets separate: - -| Path | Purpose | -|------|---------| -| `eva/` | Importable model architecture and tokenizer code | -| `tools/` | CLI entry points and reusable generation/scoring utilities | -| `config/` | YAML examples for generation, scoring, and directed evolution | -| `scripts/` | Docker-oriented shell wrappers | -| `notebooks/` | Figure reproduction, design, prediction, and interpretability notebooks | -| `data/test_data/` | Small bundled examples for smoke checks | -| `data/output/` | Local output location for examples | -| `checkpoint/` | Local model checkpoint directory; download separately | - -Large model checkpoints and datasets are intentionally not packaged in the Python wheel. Keep them in `checkpoint/` or another explicit path and pass that path with `--checkpoint`. - -
- -## Condition Control - -EVA supports conditioning on **RNA type** and **species/lineage** for both generation (`generate.py`) and scoring (`predict.py`). These conditions can be used independently or combined. - -### RNA Types - -| RNA Type | Description | -|----------|-------------| -| mRNA | Messenger RNA - carries genetic information from DNA to ribosomes | -| tRNA | Transfer RNA - brings amino acids to the ribosome during translation | -| rRNA | Ribosomal RNA - forms the core of the ribosome structure | -| miRNA | MicroRNA - regulates gene expression | -| lncRNA | Long non-coding RNA - various regulatory functions | -| circRNA | Circular RNA - circularized RNA molecules | -| snoRNA | Small nucleolar RNA - modifies other RNAs | -| snRNA | Small nuclear RNA - involved in splicing | -| piRNA | PIWI-interacting RNA - silences transposons | -| sRNA | Small RNA - general category for small RNA molecules | -| viral_RNA | RNA from viruses | - -### Species/Lineage - -Species can be specified in three ways: `--taxid`, `--species`, or `--lineage` (Greengenes format). - -Common species: - -| TaxID | Species | -|-------|---------| -| 9606 | Homo sapiens (Human) | -| 10090 | Mus musculus (Mouse) | -| 10116 | Rattus norvegicus (Rat) | -| 7227 | Drosophila melanogaster (Fruit fly) | -| 6239 | Caenorhabditis elegans (Nematode) | -| 3702 | Arabidopsis thaliana (Plant) | -| 4932 | Saccharomyces cerevisiae (Yeast) | -| 562 | Escherichia coli (Bacteria) | - -
- -## Generation - -**10x Higher Modeling Accuracy Than Ever Before — At Both Sequence and Structure Level.** - - - - - - -
- RNA Landscape Modeling Comparison -
RNA landscape modeling comparison - 		- Species-specific RNA Landscape Modeling Comparison -
Species-specific modeling comparison -
- -### CLM - -CLM (Causal Language Model) generates RNA sequences autoregressively from left to right. This is the primary generation mode in EVA. - -#### Unconditional Generation - -Generate sequences without any biological constraints: - -```bash -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --num_seqs 1000 \ - --output /output/unconditional.fa -``` - -#### Conditional Generation - -EVA supports conditioning on **RNA type**, **species** (via TaxID, species name, or lineage string), or both. See [Condition Control](#condition-control) for the full list of supported RNA types and species. - -
- RNA Type Generation -
- -```bash -# RNA type only -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --rna_type mRNA \ - --num_seqs 1000 \ - --output /output/mrna.fa - -# Species only (via TaxID) -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --taxid 9606 \ - --num_seqs 1000 \ - --output /output/human.fa - -# Both RNA type and species -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --rna_type mRNA \ - --taxid 9606 \ - --num_seqs 1000 \ - --output /output/human_mrna.fa -``` - -Species can also be specified via `--species homo_sapiens` or `--lineage "D__Eukaryota;P__Chordata;..."` in Greengenes format. - -
- Species Generation -
- -#### Continuation Mode - -Extend existing sequences in either direction. Use `--split_ratio` (fraction) or `--split_pos` (exact position) to control the split point. - -**Forward** (extend 3' end): +For model inference, build and enter the runtime from the repository root: ```bash -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --input /input/partial_seq.fa \ - --direction forward \ - --split_ratio 0.5 \ - --num_seqs 5 \ - --output /output/continuation.fa +mkdir -p checkpoint results +docker build -f docker/Dockerfile -t eva:local . +docker run --rm -it --gpus device=0 --name eva-repro \ + -v "$PWD":/eva -w /eva eva:local bash ``` -**Reverse** (extend 5' end): +Then follow the GPU example below **inside the container**. Full setup, source +installation and HPC instructions are in [Installation](docs/INSTALLATION.md). -```bash -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --input /input/partial_seq.fa \ - --direction reverse \ - --split_pos 699 \ - --num_seqs 20 \ - --output /output/reverse_continuation.fa -``` - -Add `--output_details` to include prompt, ground truth, and generated content in the output. - -### GLM - -GLM (General Language Model) performs span infilling — it masks a region within an existing sequence and generates what should fill the gap based on surrounding context. Like CLM, GLM supports both unconditional and conditional generation. - -#### Unconditional Infilling +## Reproducing the paper -Fill in a masked region without any biological constraints: +The complete representative example downloads a pinned public checkpoint, +checks its files, scores all 135 Milena sequences, calculates the metric and +writes diagnostic plots: ```bash -eva-generate \ - --checkpoint /path/to/model \ - --format glm \ - --input /input/sequences.fa \ - --span_ratio 0.1 \ - --num_seqs 5 \ - --output /output/glm_output.fa +python scripts/download_reproduction_checkpoint.py \ + --model EVA_1.4B_CLM --destination checkpoint +python scripts/reproduce_milena.py \ + --checkpoint checkpoint/EVA_1.4B_CLM --output results/milena_fresh ``` -#### Conditional Infilling +Observed new-inference Spearman: **0.8394237924835843**; archived reference: +**0.8360456283218484**. The difference remains visible in the output and is not +an execution failure. Add `--strict-reference` only to request a nonzero exit +when comparison at the stored reference precision fails. Input, dependency, +scoring and plotting failures always remain errors. -Condition on RNA type and/or species to generate biologically consistent infills: +The recorded A100 run scored the assay in about **12 seconds**, with **2.94 GiB** +peak PyTorch-allocated GPU memory. These are measurements for this example, +not minimum hardware requirements or estimates for other workloads; downloads, +loading, hashing and plotting add overhead. -```bash -eva-generate \ - --checkpoint /path/to/model \ - --format glm \ - --input /input/sequences.fa \ - --rna_type mRNA \ - --taxid 9606 \ - --span_ratio 0.2 \ - --num_seqs 5 \ - --output /output/glm_conditional.fa -``` +- [Reproduction guide](docs/REPRODUCTION.md): commands, inputs, outputs and validation scope. +- [Reviewer requirements](docs/REVIEWER_REQUIREMENTS.md): six software requirements and supporting evidence. +- [Paper workflow coverage](docs/PAPER_WORKFLOWS.md): supported entry points and missing resources. +- [Validation and release status](docs/RELEASE_STATUS.md): dated evidence and publication state. -#### Span Parameters +## Model and data resources -| Parameter | Description | Example | -|-----------|-------------|---------| -| `--span_length` | Fixed number of nucleotides to mask | `--span_length 20` | -| `--span_ratio` | Fraction of sequence to mask | `--span_ratio 0.1` | -| `--span_position` | Where to place the span: `random` or specific index | `--span_position 100` | -| `--span_id` | Which span token to use: `random` or 0-49 | `--span_id 0` | +The [resource index](docs/RESOURCES.md) links checkpoints, OpenRNA and deposited +analysis data. The representative example fixes the model revision and SHA256 +checksums in its manifest. Historical competitor resources and available +execution entry points are listed in [comparison models](docs/OFFICIAL_MODEL_RESOURCES.md). -### Sampling Parameters +## Repository layout -| Parameter | Description | Recommended Range | -|-----------|-------------|------------------| -| `--temperature` | Controls randomness. Lower = more deterministic, higher = more diverse | 0.1 - 1.5 | -| `--top_k` | Only consider the top k most likely nucleotides at each position | 10 - 100 | -| `--top_p` | Nucleus sampling — consider smallest set of nucleotides whose cumulative probability exceeds p | 0.8 - 0.95 | +| Directory | Contents | +|---|---| +| `eva/`, `tools/` | Model, tokenizer and inference CLIs | +| `training/`, `finetune/`, `config/` | Training entry points and configurations | +| `scripts/` | Portable reproduction tools and shell entry points | +| `notebooks/` | Analysis notebooks and benchmark inputs | +| `reproduction/` | Versioned inputs, historical sources and validation records | +| `docs/`, `tests/` | Guides and regression tests | -Example with all sampling parameters: +The Python wheel contains the importable model and CLI packages. Use the full +source checkout for training, notebooks and paper-reproduction inputs. +Historical sources under `reproduction/` are labeled separately from supported +entry points and may preserve old paths. -```bash -eva-generate \ - --checkpoint /path/to/model \ - --format clm \ - --temperature 0.8 \ - --top_k 50 \ - --top_p 0.9 \ - --num_seqs 100 \ - --output /output/sampled.fa -``` +## Development and support -
+See [CONTRIBUTING.md](CONTRIBUTING.md) for installation, tests and contribution +instructions. [Open an issue](https://github.com/GENTEL-lab/EVA/issues/new/choose) +with your commit, environment, command and error log when reporting a problem. +CPU checks run on pull requests and updates to main; GPU validation is recorded +separately and is not implied by a CPU check. -## Scoring - -EVA achieves state-of-the-art zero-shot fitness prediction across both mRNA and ncRNA benchmarks, substantially outperforming existing RNA, codon, and DNA language models — including models with up to 40B parameters. This strong sequence-level understanding directly translates into high-quality generation: EVA's learned distribution faithfully captures the evolutionary constraints of natural RNA, enabling it to produce biologically realistic sequences out of the box. +## Citation - - - - - -
- Zero-shot mRNA Fitness Prediction - - Zero-shot ncRNA Fitness Prediction -
+Software citation metadata are in [CITATION.cff](CITATION.cff). Cite the exact +Git commit used for your work and the paper linked above. The integrated +**1.2.0rc1** source is public on main; its formal release and archival code DOI +are pending. Existing data DOIs identify data deposits, not this code revision. -Evaluate how well a given sequence fits the model's learned distribution by computing its log-likelihood. Higher (less negative) scores indicate more probable sequences. +## License -### RNA Mode +Source code is released under [Apache-2.0](LICENSE). External datasets and +third-party checkpoints retain their own upstream terms. -Score RNA sequences and compute per-sequence log-likelihood: +## Detailed guides and previous links -```bash -eva-predict \ - --checkpoint /path/to/model \ - --input /input/sequences.fa \ - --output /output/scores.json -``` +The detailed reference has moved into the guides below. Existing README +section links remain available here. -Supports `--rna_type` and `--taxid` conditioning, same as generation. + **Why Use EVA?** → [Open guide](docs/REPRODUCTION.md) -### Protein Mode + **Key Modules** → [Open guide](docs/REPRODUCTION.md) -Score protein sequences by reverse-translating them to RNA first: + **Our Journey with EVA Starts Here 👋** → [Open guide](docs/REPRODUCTION.md) -```bash -eva-predict \ - --checkpoint /path/to/model \ - --input /input/proteins.fa \ - --output /output/protein_scores.json \ - --mode protein \ - --codon_optimization first -``` + **Option A: Local source install** → [Open guide](docs/REPRODUCTION.md) -`--codon_optimization` options: `first` (first codon in table) or `most_frequent` (most common codon for the species). + **Option B: Docker runtime** → [Open guide](docs/REPRODUCTION.md) -
+ **Option C: Singularity / Apptainer (HPC clusters)** → [Open guide](docs/REPRODUCTION.md) -## Directed Evolution + **Troubleshooting: Triton `undefined symbol: cuModuleGetFunction`** → [Open guide](docs/INSTALLATION.md#troubleshooting-triton-undefined-symbol-cumodulegetfunction) -EVA supports in-silico directed evolution — an iterative optimization pipeline that improves a given RNA sequence through cycles of mutation, LLM-based fitness scoring, and structural stability evaluation. Starting from an existing RNA sequence, EVA applies point mutations at specified positions, scores each mutant by log-likelihood and Minimum Free Energy (MFE via LinearFold), and uses simulated annealing with dynamic beam search to balance exploration and exploitation. Over multiple rounds the temperature cools to converge on optimized sequences, and the top N candidates ranked by combined LLM + MFE score are returned. + **Pre-release smoke test** → [Open guide](docs/INSTALLATION.md#pre-release-smoke-test) - - - - - - - - - -
- circRNA Illustration - - mRNA Optimization Illustration -
- circRNA Optimization - - mRNA Optimization -
+ **Model download** → [Open guide](docs/REPRODUCTION.md) -### Usage + **Release and Versioning** → [Open guide](docs/RELEASE_STATUS.md) -```bash -eva-evolve \ - --checkpoint /path/to/model \ - --input /input/sequence.fa \ - --output /output/evolved.fa \ - --rna_type mRNA \ - --taxid 9606 \ - --iterations 10 \ - --mutations 5 \ - --beam_width 10 \ - --output_count 5 -``` + **Repository Paths** → [Open guide](docs/REPRODUCTION.md) -### Key Parameters - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `--iterations` | Number of evolution cycles | 10 | -| `--mutations` | Total mutations per iteration | 2 | -| `--beam_width` | Beam search width — keeps top candidates after each position | 10 | -| `--output_count` | Number of final sequences to output | 5 | -| `--T_init` / `--T_min` | Simulated annealing temperature range | 1.0 → 0.01 | -| `--cooling_rate` | Temperature decay per iteration | 0.95 | -| `--mutate_positions` | Specific positions to mutate (0-based, comma-separated) | Random | -| `--mutate_range` | Range of positions to mutate (e.g., `0-100`) | Entire sequence | - -
- -## Batch Processing with YAML - -Define multiple tasks in a single YAML config file. The `defaults` section sets shared parameters, which individual tasks can override. - -### Generation Config Example - -```yaml -checkpoint: /path/to/model -output_dir: ./output - -defaults: - temperature: 1.0 - top_k: 50 - max_length: 8192 - batch_size: 1 - -tasks: - - name: unconditional - mode: generation - format: clm - num_seqs: 1000 - - - name: human_mrna - mode: generation - format: clm - rna_type: mRNA - taxid: "9606" - lineage: "D__Eukaryota;P__Chordata;C__Mammalia;O__Primates;F__Hominidae;G__Homo;S__Homo sapiens" - num_seqs: 1000 - - - name: glm_infill - mode: generation - format: glm - input: ./input/seqs.fa - span_ratio: 0.1 - num_seqs: 5 -``` + **Condition Control** → [Open guide](docs/USAGE.md#condition-control) -### Scoring Config Example - -```yaml -checkpoint: /path/to/model -output_dir: ./scores - -defaults: - batch_size: 128 - -tasks: - - name: score_basic - mode: scoring - input: ./input/seqs.fa - - - name: score_human_mrna - mode: scoring - input: ./input/seqs.fa - rna_type: mRNA - taxid: "9606" - normalize: true - exclude_special_tokens: true - - - name: score_protein - mode: scoring - input: ./input/proteins.fa - scoring_mode: protein - codon_optimization: first -``` + **RNA Types** → [Open guide](docs/USAGE.md#rna-types) -### Running + **Species/Lineage** → [Open guide](docs/USAGE.md#specieslineage) -```bash -eva-generate --config config.yaml # Run all tasks -eva-generate --config config.yaml --task name # Run specific task -eva-generate --config config.yaml --device cuda:1 # Override device -``` + **Generation** → [Open guide](docs/USAGE.md#generation) -
+ **CLM** → [Open guide](docs/USAGE.md#clm) -## Input/Output Formats + **Unconditional Generation** → [Open guide](docs/USAGE.md#unconditional-generation) -### Input — FASTA + **Conditional Generation** → [Open guide](docs/USAGE.md#conditional-generation) -``` ->sequence_id_1 -AUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCU ->sequence_id_2 -AUGAAAAUGCGGCCGCAUUACGUAAACGGCCGCAAAUGUUUCCGGCAAA -``` + **Continuation Mode** → [Open guide](docs/USAGE.md#continuation-mode) -### Output — Generation (FASTA) + **GLM** → [Open guide](docs/USAGE.md#glm) -``` ->unconditional_0 -AUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCU -``` + **Unconditional Infilling** → [Open guide](docs/USAGE.md#unconditional-infilling) -With `--output_details` (GLM / Continuation): + **Conditional Infilling** → [Open guide](docs/USAGE.md#conditional-infilling) -``` ->test_seq_sample0_forward_split50 -PROMPT: AUGCGCUAUGCGCUAUGCG -GROUND_TRUTH: CU AUGCGCUAUGCG -GENERATED: CU AAUGCGCUAGCG -FULL_SEQ: AUGCGCUAUGCGCUAAUGCGCUAGCG -``` + **Span Parameters** → [Open guide](docs/USAGE.md#span-parameters) -### Output — Scoring (JSON) - -```json -{ - "scores": [ - { - "header": "seq1", - "sequence": "AUGGCCGUAGU...", - "length": 67, - "log_likelihood": -1.25 - } - ] -} -``` + **Sampling Parameters** → [Open guide](docs/USAGE.md#sampling-parameters) -Higher (less negative) `log_likelihood` = better sequence. + **Scoring** → [Open guide](docs/USAGE.md#scoring) -### Output — Directed Evolution (FASTA) + **RNA Mode** → [Open guide](docs/USAGE.md#rna-mode) -Output is in FASTA format with scores in the header: + **Protein Mode** → [Open guide](docs/USAGE.md#protein-mode) -``` ->candidate_1_score=-125.4321_best -AUGC... (evolved sequence) ->candidate_2_score=-128.1056 -AUGC... (second best) -``` + **Directed Evolution** → [Open guide](docs/USAGE.md#directed-evolution) -
+ **Usage** → [Open guide](docs/USAGE.md#usage) -## Data Availability + **Key Parameters** → [Open guide](docs/USAGE.md#key-parameters) -Some large files are not included in this repository due to size constraints. They can be downloaded from the sources listed below. + **Batch Processing with YAML** → [Open guide](docs/USAGE.md#batch-processing-with-yaml) -### Model & Environment + **Generation Config Example** → [Open guide](docs/USAGE.md#generation-config-example) -| Resource | Description | Size | Source | -|----------|-------------|------|--------| -| Model Checkpoint | EVA 1.4B (MoE) pre-trained weights | | [HuggingFace](https://huggingface.co/GENTEL-Lab/EVA) | -| Docker Image (`eva_latest.tar.gz`) | Pre-built EVA runtime environment | | [Zenodo](https://doi.org/10.5281/zenodo.18786989) | -| Training Data | OpenRNA v1 — 114M full-length RNA sequences | | [HuggingFace](https://huggingface.co/datasets/GENTEL-Lab/OpenRNA-v1-114M) | + **Scoring Config Example** → [Open guide](docs/USAGE.md#scoring-config-example) -### Experiment & Benchmark Data + **Running** → [Open guide](docs/USAGE.md#running) -| Resource | Description | Size | Source | -|----------|-------------|------|--------| -| mRNA Codon Optimization | Optimized sequences from 9 vaccine targets (5 mRNA: linear, HIV, PR8, RABV, VZV; 4 circRNA: CAR-T, RABV-G, SARS-Group-I, SARS-T4) using EVA (homo/nolineage), CodonFM-1B, Evo2-1B, GemoRNA, and random baseline | 26 KB | [Zenodo](https://doi.org/10.5281/zenodo.19027142) | -| mRNA 6 Species | Generated & natural mRNA sequences for 6 species (Homo sapiens, Mus musculus, Rattus norvegicus, Cricetulus griseus, Drosophila melanogaster, Caenorhabditis elegans) with MFE, GC content, k-mer, loop-helix, pair metrics | 38 MB | [Zenodo](https://doi.org/10.5281/zenodo.19027142) | -| GeneRRNA Generation | EVA-generated rRNA sequences benchmarked against GeneRRNA database (100k natural, 1k sampled) with k-mer, loop-helix, pair, merged MFE-GC metrics | 13 MB | [Zenodo](https://doi.org/10.5281/zenodo.19027142) | -| Generated RNA Sequences | 15 RNA types (mRNA, lncRNA, circRNA, tRNA, rRNA, miRNA, piRNA, sRNA, snRNA, snoRNA, scaRNA, vault_RNA, Y_RNA, ribozyme, viral) generated by EVA with matched natural controls (5k each) and MFE, GC, k-mer, loop-helix, pair metrics | 156 MB | [Zenodo](https://doi.org/10.5281/zenodo.19027142) | + **Input/Output Formats** → [Open guide](docs/USAGE.md#inputoutput-formats) + **Input — FASTA** → [Open guide](docs/USAGE.md#input--fasta) -
+ **Output — Generation (FASTA)** → [Open guide](docs/USAGE.md#output--generation-fasta) -## Citation + **Output — Scoring (JSON)** → [Open guide](docs/USAGE.md#output--scoring-json) -Software citation metadata are in [CITATION.cff](CITATION.cff). Record the exact -version and Git commit used for an experiment. This local candidate is version -1.2.0rc1 and has not yet received an archival code DOI; the DOI and final -manuscript citation will be added after their corresponding publications. + **Output — Directed Evolution (FASTA)** → [Open guide](docs/USAGE.md#output--directed-evolution-fasta) -
+ **Data Availability** → [Open guide](docs/RESOURCES.md) -## License + **Model & Environment** → [Open guide](docs/RESOURCES.md) -This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details. + **Experiment & Benchmark Data** → [Open guide](docs/RESOURCES.md) diff --git a/docs/BENCHMARK_PROTOCOLS.md b/docs/BENCHMARK_PROTOCOLS.md index aa9d310..48823cd 100644 --- a/docs/BENCHMARK_PROTOCOLS.md +++ b/docs/BENCHMARK_PROTOCOLS.md @@ -96,3 +96,11 @@ The recovered 21M pretrain `checkpoint-31006` and midtrain `checkpoint_86006`, a Both recovered 21M stage configs set `use_lineage_prefix: true`. Therefore they do not substantiate a blanket statement that pretraining used RNA-type only and lineage began solely at midtraining. The final 1.4B mid config names a v31 pretraining checkpoint for which the matching pretraining config remains unresolved. Clarify the run-to-manuscript linkage rather than changing training objectives to match prose. The native `model_dense` implementation is still missing; neither an MoE alias nor a smoke test substitutes for it. Training data paths in historical YAMLs are provenance, not evidence that the original training datasets are publicly accessible. The separately recovered `RNAVerse/checkpoint/clm` metadata supplies a fifth concrete configuration (v4 midtraining from v24 pretrain step 15000): it also specifies mixed mode, GLM probability 0.333, EP=4, DP=4, lineage and RNA-type prefixes. It establishes another observed historical run, not a reason to conflate the v24 and v31 checkpoint families. + +## Resource recovery update: September 9, 2026 + +The [current resource index](OFFICIAL_MODEL_RESOURCES.md#recovered-comparison-model-files-september-9-2026) +records recovered RNA-FM/RNABERT/RNA-MSM snapshots and additional cached +CodonFM, ERNIE-RNA, AIDO.RNA and GenerRNA file hashes. Public download/runtime +bindings remain incomplete. These new identities supplement the historical +requirements above; they do not imply newly executed competitor benchmarks. diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md new file mode 100644 index 0000000..220314b --- /dev/null +++ b/docs/INSTALLATION.md @@ -0,0 +1,132 @@ +# Installation + +Use the complete source checkout for paper workflows. The Python wheel includes +`eva` and the inference CLIs; training scripts, notebooks and benchmark inputs +are source-checkout resources. + +## Choose an environment + +| Task | Environment | What is verified | +|---|---|---| +| Recalculate the bundled Milena metric | Python 3.10 or 3.11; standard library only | No model or GPU needed; [CPU example](REPRODUCTION.md#cpu-recalculate-stored-predictions) | +| Package/CLI use and CPU development tests | Python 3.10 or 3.11; editable install with the required extras | Package installation and CPU tests; not full MoE inference | +| Model inference and training | Linux, NVIDIA GPU and the Docker runtime below | PyTorch 2.5.1/CUDA 12.4; validated on A100 | + +## GPU runtime: Docker + +Install Docker with NVIDIA Container Toolkit on the host. Run these commands +from the repository root **on the host**: + +```bash +mkdir -p checkpoint results +docker build -f docker/Dockerfile -t eva:local . +docker run --rm -it --gpus device=0 --name eva-repro \ + -v "$PWD":/eva -w /eva eva:local bash +``` + +Inside the container, follow the [GPU benchmark](REPRODUCTION.md#gpu-run-the-complete-milena-benchmark) +or [training example](REPRODUCTION.md#training-and-fine-tuning). `cuda:0` refers +to the GPU visible inside the container. Use `exit` to return to the host. +Each output command needs a new directory; existing results are never overwritten. + +The recipe uses Python 3.11, PyTorch 2.5.1, CUDA 12.4, Transformers 4.55.0, +MegaBlocks 0.7.0 and grouped-GEMM 0.1.6. It compiles the required MoE extensions; +the first build may take 30–60 minutes, depending on hardware and networking. +Attention uses PyTorch SDPA and does not require a separate FlashAttention package. +The exact dependency pins are in `docker/requirements.txt`; the validated image +and base-image digest are recorded in [validation status](RELEASE_STATUS.md). +The recipe supports CUDA build targets 7.5, 8.0 and 9.0, but that build setting +is not a claim of runtime testing on every GPU architecture. + +If the default Python package index is unreachable, explicitly select a mirror: + +```bash +docker build --build-arg PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple \ + -f docker/Dockerfile -t eva:local . +``` + +Package versions stay pinned. Record the chosen index with the build log. + +## Source installation and CPU development + +From the repository root, use a Python 3.10 or 3.11 virtual environment: + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install -e '.[benchmark,design,dev,notebook]' +python -m pip check +python -m pytest -q tests +``` + +The CPU test suite does not require the compiled MoE extensions. For inference, +use the Docker runtime or install its exact compiled dependencies in a compatible +CUDA environment. A successful `--help` command checks installation only. + +The lightweight install `python -m pip install -e .` supplies the main package. +Extras add benchmark downloads (`benchmark`), ViennaRNA (`design`), plotting and +notebooks (`notebook`), or development checks (`dev`). No model weights are +included in the wheel. See [resources](RESOURCES.md) for checkpoint downloads. + +## Singularity / Apptainer (HPC clusters) + + +HPC users usually cannot run Docker. Convert the image with the definition +file on any machine that has Docker: + +```bash +docker build -f docker/Dockerfile -t eva:local . +singularity build eva_latest.sif docker/EVA.def +``` + +The [legacy image deposit](RESOURCES.md#runtime-image) is not the current +validated source build. For this revision, build the local Docker image above +and convert it with the supplied definition file. + +Always launch with `--nv` so the host NVIDIA driver is available: + +```bash +singularity shell --nv eva_latest.sif +singularity exec --nv --bind /path/to/checkpoint:/eva/checkpoint \ + eva_latest.sif python /eva/tools/generate.py --help +``` + +Verify the runtime before running notebooks or training: + +```bash +singularity exec --cleanenv --nv eva_latest.sif \ + python -c "from triton.runtime import driver; print(driver.active)" +``` + +### Troubleshooting: Triton `undefined symbol: cuModuleGetFunction` + +Triton 3.1.0 compiles `cuda_utils.so` on first use and links it against +`-lcuda` at link time, resolving through the ldconfig cache rather than +`LD_LIBRARY_PATH`. Images built via `docker commit` on GPU hosts contain +0-byte `libcuda.so.*` placeholders: Docker `--gpus` mounts the real driver +over them (masking the bug), while Singularity `--nv` leaves them visible, +so the linker binds against an empty file and every Triton import fails +with `undefined symbol: cuModuleGetFunction`. + +Images built from the current repository fix this at the source: the +placeholders are removed at build time, and `eva/_runtime_env.py` points +Triton at the injected driver via `TRITON_LIBCUDA_PATH` on every +`import eva`. Users of images released before this fix must rebuild from +the current repository. + +Verify after any launch method: + +```bash +python -c "from triton.runtime import driver; print(driver.active)" +``` + +### Pre-release smoke test + +When validating an image for an HPC environment, test both launch methods. +The published validation record covers Docker; verify Singularity/Apptainer +on your target cluster before relying on it: + +```bash +docker/smoke_test.sh docker eva:local +docker/smoke_test.sh singularity eva_latest.sif +``` diff --git a/docs/OFFICIAL_MODEL_RESOURCES.md b/docs/OFFICIAL_MODEL_RESOURCES.md index 2679a36..58a83e7 100644 --- a/docs/OFFICIAL_MODEL_RESOURCES.md +++ b/docs/OFFICIAL_MODEL_RESOURCES.md @@ -7,13 +7,13 @@ execution gaps are listed in [PAPER_WORKFLOWS.md](PAPER_WORKFLOWS.md). | Method | Upstream resources | Environment boundary | |---|---|---| -| RNA-FM, RNABERT, RNA-MSM | Historical configuration uses the Multimolecule model ports; the three recorded snapshot revisions currently return HTTP 404 | Recover the matching checkpoint plus tokenizer and Multimolecule package version before inference. | +| RNA-FM, RNABERT, RNA-MSM | [RNA-FM upstream](https://github.com/ml4bio/RNA-FM); the historical Multimolecule ports use three revisions that still return HTTP 404 | Matching cached checkpoint/tokenizer hashes are now recorded below; public download and the original Multimolecule runtime remain unresolved. | | GENA-LM | [Official code and models](https://github.com/AIRI-Institute/GENA_LM) | DNA tokenizer/custom-code runtime; recorded snapshot resolves but model-name binding is unresolved. | | ERNIE-RNA | [Official repository and checkpoint instructions](https://github.com/Bruce-ywj/ERNIE-RNA/blob/main/README.md) | Use its fairseq environment and original pretrained checkpoint, not a structure fine-tuned model. | | Evo2 | [Official models and runtime](https://github.com/arcinstitute/evo2) | Separate Vortex/Transformer Engine environment; exact paper model and runtime revisions remain required. | -| CodonFM | [Official repository and model downloads](https://github.com/NVIDIA-BioNeMo/CodonFM) | Its own Docker recipe and codon tokenizer; original and accelerated implementations must not be interchanged without a protocol check. | +| CodonFM | [Official repository and model downloads](https://github.com/NVIDIA-BioNeMo/CodonFM) | Cached 600M/1B weights and config hashes are recorded below; public revision/runtime linkage is still required. | | AIDO.RNA | [ModelGenerator](https://github.com/genbio-ai/ModelGenerator), [AIDO model collection](https://github.com/genbio-ai/AIDO) | Match the paper backbone and ModelGenerator API version; current releases do not prove compatibility with the historical worker. | -| GenerRNA | [Official model checkpoint and tokenizer](https://huggingface.co/pfnet/GenerRNA) | Pair model_updated.pt with its corresponding tokenizer and architecture. Original file hashes remain needed. | +| GenerRNA | [Official model checkpoint and tokenizer](https://huggingface.co/pfnet/GenerRNA) | Cached model_updated.pt and tokenizer-config hashes are recorded below; the complete tokenizer/runtime and public-version binding remain required. | | CodonGPT | The paper-linked NanilTx/codonGPT_pub repository returned HTTP 404 in this check | Obtain the original repository/version and weights; a similarly named software project is not a substitute. | | ESM | [Official model downloads and fair-esm environment](https://github.com/facebookresearch/esm) | Recover exact model size/ensemble and original ProteinGym reference/DMS inputs. | | ProGen3 | [Official models, Docker runtime and scoring CLI](https://github.com/Profluent-AI/progen3) | Official scorer averages forward/reverse likelihood; verify this against the original EVA comparison protocol and fix the corresponding weight revision. | @@ -21,3 +21,26 @@ execution gaps are listed in [PAPER_WORKFLOWS.md](PAPER_WORKFLOWS.md). `reproduction/release/competitor_resource_verification.json` records the read-only Hugging Face revision checks and available LFS object hashes. No third-party weight download or inference is claimed by this resource index. + +## Recovered comparison-model files: September 9, 2026 + +The original RNA-FM, RNABERT and RNA-MSM cache snapshots were recovered and +all 15 weight/config/tokenizer files were hashed. Their model configurations +record Transformers 4.50.0; the exact Multimolecule runtime is still unbound. +The [recovered identity record](../reproduction/release/recovered_rna_snapshots_20260909.json) +contains each historical revision and every file SHA256. Ready-to-use manifests +for the existing `--model-manifest` interface are in +`reproduction/release/model_manifests/`. Keep a manifest outside its model folder. + +The historical public revisions still return 404. Current public weights have +different file hashes; no tensor-level equivalence is claimed and no current +weight is substituted. The recovered files are not rehosted in this source +repository. Original environment/protocol correspondence and a working public +download remain required before these methods can be claimed fully reproducible. + +The [additional recovered-asset record](../reproduction/release/recovered_competitor_assets_20260909.json) +adds 13 file identities from CodonFM 600M/1B, ERNIE-RNA, AIDO.RNA and GenerRNA +caches. These include configurations and the ERNIE environment-file hash; the +two ERNIE copies have identical hashes. A local cached weight is not itself a +public download or proof that its full environment/assay protocol is recovered. +No additional competitor inference was performed in this repository update. diff --git a/docs/PAPER_WORKFLOWS.md b/docs/PAPER_WORKFLOWS.md index 9b32242..6089152 100644 --- a/docs/PAPER_WORKFLOWS.md +++ b/docs/PAPER_WORKFLOWS.md @@ -31,11 +31,11 @@ The entries below distinguish recovered version identifiers from missing ones. | RNABERT | `multimolecule/rnabert`, revision `7a9b5d5d5a96931ff056828c0968479098506cea` | Same bounded masked-LM entry; historical revision currently returns HTTP 404. | | RNA-MSM | `multimolecule/rnamsm`, revision `ad61a4915674596557b3c3c4fe4fd7162d36acc8` | Historical revision currently returns HTTP 404; sequence/MSA protocol correspondence is also unverified. | | GENA-LM | Local snapshot names `AIRI-Institute/gena-lm-bert-base-t2t-multi`, revision `4633e5a1ada905bb7afee6877d71cc12578a95a5` | Original model-name field differs from local snapshot name. Resolve that binding before downloading or running a replacement. | -| ERNIE-RNA | `ERNIE-RNA_pretrain.pt` | Preserved ERNIE adapter and worker; exact release/checksum and fairseq environment still required. | +| ERNIE-RNA | `ERNIE-RNA_pretrain.pt` | Preserved ERNIE adapter and worker; cached weight and environment-file hashes are now recorded; matching public release and runtime execution remain unverified. | | Evo2 1B, 7B, 40B | `evo2_1b_base`, `evo2_7b_base`, `evo2_40b_base` | Preserved Evo adapter; exact revisions and official compatible runtime still required. Do not replace the original scoring objective. | -| CodonFM 600M and 1B | Original 600m/1b model selectors | Preserved `score_codonfm.py` and adapter; original container and immutable weight identities remain missing. | -| AIDO.RNA | Original model-weight folder and ModelGenerator framework | Preserved AIDO worker/adapter; exact model snapshot and environment still required. | -| GenerRNA | `model_updated.pt` and separate tokenizer | Preserved GenerRNA worker/adapter; checkpoint/tokenizer checksums and matching runtime still required. | +| CodonFM 600M and 1B | Original 600m/1b model selectors | Preserved `score_codonfm.py` and adapter; cached weight/config identities are now recorded; original runtime and public version binding remain unresolved. | +| AIDO.RNA | Original model-weight folder and ModelGenerator framework | Preserved AIDO worker/adapter; cached weight-shard/config hashes are now recorded; exact public snapshot and environment binding remain required. | +| GenerRNA | `model_updated.pt` and separate tokenizer | Preserved GenerRNA worker/adapter; cached weight and tokenizer-config hashes are now recorded; complete tokenizer/runtime and public version binding remain required. | | CodonGPT | Original worker `compute_codongpt_ll.py` | Weight revision and original environment still required. | | ESM protein models | `upstream/protein/evaluate_dms_esm_domain.py` | Strict native entry `run_competitor.py protein-esm`; exact reference/DMS files and model ensemble must be supplied. | | ProGen3 and other unmatched protein rows | Existing summary-table entries | Missing producer/version must be recovered; a table row is not an inference implementation. | @@ -66,3 +66,11 @@ The recovered Evo2 7B protein row remains a separate candidate table under into the current submission table. Four ncRNA model comparisons have recorded sample-count differences of one; their original inclusion/WT rules still need to be documented before those comparisons can be called complete. + +## Resource recovery update: September 9, 2026 + +The [current resource index](OFFICIAL_MODEL_RESOURCES.md#recovered-comparison-model-files-september-9-2026) +records recovered RNA-FM/RNABERT/RNA-MSM snapshots and additional cached +CodonFM, ERNIE-RNA, AIDO.RNA and GenerRNA file hashes. Public download/runtime +bindings remain incomplete. These new identities supplement the historical +requirements above; they do not imply newly executed competitor benchmarks. diff --git a/docs/RELEASE_STATUS.md b/docs/RELEASE_STATUS.md index 533f76d..c4ad41e 100644 --- a/docs/RELEASE_STATUS.md +++ b/docs/RELEASE_STATUS.md @@ -1,41 +1,59 @@ -# Reproducibility release candidate - -This is an integrated local candidate based on public commit -`07f4cb523ed5053f1ac11d79e4c524167b00bbdd`. It has not been published or assigned -a code DOI. It combines the existing engineering fixes with the benchmark -supplement and a pinned EVA-1.4B Milena entry point. - -The current submission data are preserved as confirmed by the author. Older -provenance reports are retained as dated evidence and do not automatically -establish an error in replacement data. - -The integrated test suite passed 128 tests. Synthetic pretraining, mid-training -and fine-tuning each completed two optimizer steps with exact saved-weight -reloads. The three notebook code cells executed in an independent kernel; -its optional custom prediction section remains unconfigured. `pip check` passed. -The complete source archive was then built from the official PyTorch -2.5.1/CUDA 12.4 base (resolved image digest -`sha256:14611869895df612b7b07227d5925f30ec3cd6673bad58ce3d84ed107950e014`). -In this clean image, all 128 tests, the three installed CLI help commands, -training/save/reload, the full 135-row inference, archived SAE recalculation, -summary audit and independent-kernel notebook execution completed. `pip check` -reported no broken requirements. The build used the Aliyun PyPI mirror with -the pinned dependencies; EVA attention uses torch SDPA, so no external -flash-attn package is installed. GPU work used one idle A100 and batch size 1. - -The complete 135-row public-checkpoint Milena inference ran successfully: -Spearman 0.8394237924835843 versus 0.8360456283218484 in the current table. -All 135 new scores are identical to a separate earlier fresh run under different -PyTorch/MegaBlocks versions. The recovered original log records batch size 45; its causal role in the -score difference has not been established. -See `reproduction/milena_14b/expected/README.md` for the observed result and -successful public download and byte-identical GPU cache verification. - -Release still requires a complete public-checkpoint example with interpretable -numerical agreement and accurate documentation of all required workflow resources. -A failed or unexplained benchmark comparison prevents a claim that the -reviewer's reproducibility concerns have all been resolved. - -Exact model and input identities are in `reproduction/milena_14b/manifest.json`. -See [PAPER_WORKFLOWS.md](PAPER_WORKFLOWS.md) for missing competitor-resource -bindings and workflow limitations. No placeholder DOI is provided. +# Validation and release status + +The integrated **1.2.0rc1** source is public on `main`. The reproducibility +repairs were merged at `9b5d25ac7cb26de88420f56021f4c0eeb2052e9e`. +A formal release for this revision and an archival code DOI are pending; +the older `v1.1.1` release does not contain these repairs. + +## Recorded validation: September 9, 2026 + +The source archive at `70c6c4f7aff2cf12210d8b280c1d4569ca22c0a0` was built in a +clean Docker environment. The subsequent `9b5d25a` commit added documentation +and validation records without changing runtime code or inputs. + +| Check | Recorded result | +|---|---| +| Regression tests | 128 passed; none skipped or failed | +| Installation | `pip check` and all three installed CLI help commands passed | +| Synthetic training | Two steps per stage; pretrain/midtrain/fine-tune weights saved and reloaded exactly | +| Notebook | Three code cells executed in an independent kernel; optional custom predictions unconfigured | +| Milena inference | All 135 samples scored from the pinned public checkpoint | +| Archived calculations | Milena metric, SAE calculations and benchmark summary audit executed | + +`reproduction/release/clean_validation.json` records the commit and hashes. +The image used PyTorch 2.5.1/CUDA 12.4, Transformers 4.55.0 and MegaBlocks 0.7.0. +The base-image digest was +`sha256:14611869895df612b7b07227d5925f30ec3cd6673bad58ce3d84ed107950e014`. +Installation used the Aliyun PyPI mirror with pinned versions. GPU validation +used one idle A100 and batch size 1. This historical record does not automatically +validate later changes; current CPU checks run separately in GitHub Actions. + +## Representative benchmark result + +New Milena inference gives **0.8394237924835843**, compared with the stored +**0.8360456283218484**. The **0.0033781641617359748** difference is accepted for +this representative example and is not a merge or release blocker. Both +numbers and all predictions remain available. No input labels, scoring +objective or paper metric have been changed to reduce the difference. + +[Reproduction](REPRODUCTION.md) documents normal completion and the optional +`--strict-reference` diagnostic. [Expected outputs](../reproduction/milena_14b/expected/README.md) +record the measurements. Historical batch-size observations are context, not +a demonstrated cause of the difference. + +## Remaining coverage and publication work + +- Recovered comparison-model hashes are in the [resource index](OFFICIAL_MODEL_RESOURCES.md). + Public downloads, original runtimes or producers remain missing for some methods; + cached artifact identities alone do not close those gaps. +- Training machinery is tested at small scale. Original dense-model code and + some paper-specific training/run bindings are not available. +- Some figure-specific input/command mappings and complete SAE generation + outputs remain outside the bundled reproducible workflows. +- Formal release, code DOI and the corresponding manuscript citation remain + separate steps. No placeholder DOI is provided. + +The [six reviewer requirements](REVIEWER_REQUIREMENTS.md) and +[paper workflow table](PAPER_WORKFLOWS.md) identify the evidence and remaining +scope. Historical notes are dated background records; use current guides +for present software behavior. diff --git a/docs/REPRODUCTION.md b/docs/REPRODUCTION.md index cb97d02..84cabc1 100644 --- a/docs/REPRODUCTION.md +++ b/docs/REPRODUCTION.md @@ -1,115 +1,123 @@ # Reproducing EVA computations -This guide is the entry point for the current source candidate. It separates -fresh model inference, recalculation of stored predictions, and summary plotting. -Use the full source checkout: the Python wheel does not contain experiment inputs. -Current validation is reported in [RELEASE_STATUS.md](RELEASE_STATUS.md). +Choose one of the three workflows below. Paths are relative to the complete +source checkout; the Python wheel does not contain benchmark inputs. Use a new +output directory for each run. See [Installation](INSTALLATION.md) for setup +and [validation status](RELEASE_STATUS.md) for dated execution evidence. -## Install the full runtime +## CPU: recalculate stored predictions -The Docker recipe targets Python 3.11, PyTorch 2.5.1 with CUDA 12.4, -Transformers 4.55.0 and MegaBlocks 0.7.0. A CUDA GPU is needed for the model. +**Requirements:** Python 3.10 or 3.11; standard library only, no model or GPU. From the repository root: ```bash -docker build -f docker/Dockerfile -t eva-repro:local . -mkdir -p checkpoint results -docker run --rm -it --gpus device=0 \ - -v "$PWD":/eva -w /eva eva-repro:local bash -python -m pytest -q tests +python3 scripts/reproduce_historical_benchmark.py \ + --artifact-only --output results/milena_stored ``` -If the default package registry is unreachable, select an accessible index -with `docker build --build-arg PIP_INDEX_URL=INDEX_URL ...`; dependency versions -remain pinned. Record the selected index with the build log. +This verifies bundled hashes and recomputes the metric from all 135 archived +Milena predictions. Expected outputs: -On an existing compatible CUDA runtime, use an isolated environment and -`python -m pip install -e '.[benchmark,design,dev,notebook]'`. -This does not build MegaBlocks/grouped-GEMM; use the Docker recipe for those -compiled dependencies. EVA attention uses PyTorch SDPA. The unused external -`flash-attn` package is not required or installed by this recipe. Missing dependencies fail explicitly. +| File | Contents | +|---|---| +| `results/milena_stored/report.json` | `n: 135`, `arithmetic_match: true`, Spearman `0.8360456283218484` | +| `results/milena_stored/sequence_label_audit.csv` | 135 rows with sequence IDs, labels and stored scores | -## One complete benchmark from a public checkpoint +The explicit `--artifact-only` mode returns 0 for successful recalculation. +It does not execute a model. Historical metadata columns are retained for +traceability and do not replace the selected labels. +For table-wide summaries, see [benchmark calculations](BENCHMARK_RELEASE.md). +That separate audit can return 2 for documented coverage/sample-count issues +while still writing its reports. + +## GPU: run the complete Milena benchmark + +**Requirements:** the [Docker GPU runtime](INSTALLATION.md#gpu-runtime-docker), +one NVIDIA GPU and about 3.06 GB of downloaded weights. The measured runtime is +A100/PyTorch 2.5.1/CUDA 12.4; other hardware has not been certified. +After entering the container with the checkout mounted at `/eva`, run: ```bash python scripts/download_reproduction_checkpoint.py \ --model EVA_1.4B_CLM --destination checkpoint python scripts/reproduce_milena.py \ - --checkpoint checkpoint/EVA_1.4B_CLM --output results/milena_14b + --checkpoint checkpoint/EVA_1.4B_CLM --output results/milena_fresh ``` -The downloader fixes Hugging Face revision -`514db6705637c1ec963b728768fc9b34728699ee`. The workflow verifies the weight, -config, tokenizer and input hashes before scoring all 135 Milena sequences. -It uses the bundled historical model implementation, BF16, batch size 1, -no conditioning, `5{RNA}3`, and summed causal log likelihood excluding -`5`, `3` and `` targets. No alternative objective is chosen to fit a score. -The public config differs from the historical config only in `moe_world_size` -(1 instead of 4); single-GPU execution sets this to 1 in both cases. - -The current submission labels are retained as confirmed by the author. Their -positional association is frozen by the exact FASTA and label hashes. Header -metadata from old archives is not used to replace the experimental readouts. -The fixed protocol and resource identities are in +The download fixes Hugging Face revision +`514db6705637c1ec963b728768fc9b34728699ee`. Before inference, the runner checks +weights, config, tokenizer, FASTA and label hashes against `reproduction/milena_14b/manifest.json`. - -Outputs are `predictions.csv`, `report.json`, `inference.log`, the detailed -inference report, and `plot/` with PNG/SVG diagnostics. The report contains -Spearman, its full-precision difference from the current table, timing, GPU -memory, package versions and hashes. The current reference is -0.8360456283218484. The machine comparison uses the stored CSV precision -as a diagnostic, not as a scientific acceptance threshold. The manuscript -plot does not annotate an individual Milena coefficient. The observed delta -and archive-vector comparison are disclosed separately in the expected output. -Exit 2 means the run finished but the metric differs; -other failures raise an error and preserve a failure report. Exit 0 for -`--check-only` verifies inputs only. A numerical match alone does not establish -all scientific provenance; inspect the report and release status. - -The older `scripts/reproduce_dms.py` retains its mean-scoring behavior and -EVA-21M default for compatibility. Changing its reference row does not select -the historical 1.4B protocol. Use `reproduce_milena.py` for this example. - -## Existing predictions and summary tables - -For a CPU-only summary audit (no EVA installation required): - -```bash -python scripts/benchmark_release.py audit-reference \ - --manifest reproduction/benchmark_release/reference_manifest.json \ - --data-root reproduction/benchmark_release/reference \ - --output results/reference_audit --plot -``` - -This command currently returns 2 for documented coverage/sample-count issues; -its outputs are still written. It does not run a model. See -[BENCHMARK_RELEASE.md](BENCHMARK_RELEASE.md) for keyed prediction CSVs, -metric calculation and the notebook. Author confirmation of data correctness -does not by itself resolve an undocumented difference in sample inclusion. - -## Training and other analyses - -For a small synthetic pretraining/midtraining/fine-tuning check on one GPU: +The protocol uses all 135 samples, the bundled historical implementation, +BF16, batch size 1, no conditioning and summed causal log likelihood excluding +direction/EOS targets. The prompt is `5{RNA}3`. Input order and +sequence-to-label association are frozen by hashes. The public config changes +historical `moe_world_size` from 4 to 1; this workflow uses one GPU. + +| Output inside `results/milena_fresh/` | Contents | +|---|---| +| `predictions.csv` | 135 keyed predictions with sequence and experimental value | +| `report.json` | Metric, comparison, execution status, timing, environment and hashes | +| `inference.log`, `inference/report.json` | Backend log and detailed execution evidence | +| `plot/` | PNG/SVG diagnostics and plot report | + +### Expected result and exit codes + +The clean-environment run produced **0.8394237924835843**, compared with the +archived **0.8360456283218484**: difference **0.0033781641617359748**. This +observation is retained and accepted for the representative example; it is +not an execution failure. The original table and predictions are unchanged. +[Expected outputs](../reproduction/milena_14b/expected/README.md) give the evidence. + +That run took 12.08 seconds for inference and 23.40 seconds for the runner, +with 3,157,299,200 bytes (2.94 GiB) peak PyTorch-allocated GPU memory on an +A100-SXM4-80GB. Downloads are excluded. Allocated memory is not total device +usage or a minimum GPU specification. + +- Default: returns **0** after valid inference, metrics and plots, including + when the reference comparison differs. +- `--strict-reference`: returns **2** after an otherwise valid run if comparison + at the stored reference precision fails. This is diagnostic, not a scientific + tolerance. Use a new output directory when rerunning in strict mode. +- `--check-only`: returns **0** for input/checkpoint verification only; + `fresh_inference` remains false and no model is run. +- Missing dependencies, changed inputs/weights, invalid predictions and plotting + failures remain errors. Failures after output initialization are recorded in + `report.json`. Existing output directories are never overwritten. + +The report separates `execution_status` (`COMPLETED`, `FAILED`, +`INPUTS_VERIFIED_ONLY`) from `comparison_status` (`MATCH`, `DIFFERENCE`, `NOT_RUN`). +Existing numerical comparison fields are preserved. The legacy +`paper_result_reproduced` field is not set automatically by a successful command; +this example does not establish reproduction of every paper result. +The older `scripts/reproduce_dms.py` retains its EVA-21M/mean-scoring default. +Changing its reference row does not select the historical 1.4B protocol. + +## Training and fine-tuning + +**Requirements:** complete checkout and the GPU runtime. This small synthetic +engineering example needs no downloaded paper checkpoint: ```bash -python scripts/smoke_workflows.py --training-only --output results/training_smoke +python scripts/smoke_workflows.py \ + --training-only --output results/training_smoke ``` -This performs two optimizer steps per training stage and checks exact saved -weight reloads; it does not consume a paper checkpoint or reproduce pretraining. +It performs two optimizer steps each for pretraining, mid-training and +fine-tuning, verifies parameter updates, then saves and reloads weights. +Inspect the output report for each stage. It validates the exposed training +machinery; it does not rerun large-scale pretraining. +- [Training](../training/pretrain/README.md): stage entry points and configuration fields. +- [Fine-tuning](../finetune/aptamer/script/README.md): checkpoint, data and container paths. +- [CLI guide](USAGE.md): inputs, conditioning and task parameters. +- [Paper workflows](PAPER_WORKFLOWS.md): result/resource mapping and omitted components. +- [Reviewer requirements](REVIEWER_REQUIREMENTS.md): six requests and supporting evidence. -- [Training](../training/pretrain/README.md) and - [fine-tuning](../finetune/aptamer/script/README.md): complete-source entry - points and the synthetic forward/backward/save/reload smoke workflow. -- [Benchmark protocols](BENCHMARK_PROTOCOLS.md) and - [paper workflow coverage](PAPER_WORKFLOWS.md): competitor sources and - method-specific limitations. -- [Essentiality](ESSENTIALITY_REPRODUCTION.md): recomputation of stored - predictions with stable row identity and separate ablation fractions. -- [SAE analyses](SAE_REPRODUCTION.md): stored-result calculations and - separately identified inference demonstrations. +## Other analyses and historical sources -Do not treat small synthetic training tests as a rerun of pretraining or a -reproduction of all reported experiments. Historical manifests retain their -original paths for traceability; they are not executable installation commands. +[Benchmark protocols](BENCHMARK_PROTOCOLS.md), [essentiality](ESSENTIALITY_REPRODUCTION.md) +and [SAE analyses](SAE_REPRODUCTION.md) distinguish stored-result calculations +from inference. Sources under `reproduction/benchmark/upstream/` and +`reproduction/sae/archive/` preserve historical paths and behavior. Follow the +documented portable entry points instead of executing these archives unchanged. diff --git a/docs/RESOURCES.md b/docs/RESOURCES.md new file mode 100644 index 0000000..8e9429a --- /dev/null +++ b/docs/RESOURCES.md @@ -0,0 +1,77 @@ +# Models and data + +The model/download metadata below were read from the public APIs on +September 9, 2026. File sizes use decimal MB/GB. The machine-readable record is +`reproduction/release/public_resource_metadata_20260909.json`. + +## EVA checkpoints + +[Fixed Hugging Face revision](https://huggingface.co/GENTEL-Lab/EVA/tree/514db6705637c1ec963b728768fc9b34728699ee) +contains the following weight files. Each checkpoint also needs its matching +`config.json` and `tokenizer.json`. + +| Model directory | Weight bytes | Approximate size | Download | +|---|---:|---:|---| +| `EVA_1.4B_CLM` | 3,057,231,170 | 3.057 GB | [Files](https://huggingface.co/GENTEL-Lab/EVA/tree/514db6705637c1ec963b728768fc9b34728699ee/EVA_1.4B_CLM) | +| `EVA_1.4B_GLM` | 3,057,231,170 | 3.057 GB | [Files](https://huggingface.co/GENTEL-Lab/EVA/tree/514db6705637c1ec963b728768fc9b34728699ee/EVA_1.4B_GLM) | +| `EVA_145M` | 361,225,818 | 0.361 GB | [Files](https://huggingface.co/GENTEL-Lab/EVA/tree/514db6705637c1ec963b728768fc9b34728699ee/EVA_145M) | +| `EVA_21M` | 44,860,956 | 0.045 GB | [Files](https://huggingface.co/GENTEL-Lab/EVA/tree/514db6705637c1ec963b728768fc9b34728699ee/EVA_21M) | +| `EVA_437M` | 1,013,942,634 | 1.014 GB | [Files](https://huggingface.co/GENTEL-Lab/EVA/tree/514db6705637c1ec963b728768fc9b34728699ee/EVA_437M) | + +For the representative example, the [reproduction guide](REPRODUCTION.md) +downloads `EVA_1.4B_CLM` through the bundled downloader and verifies all three +files against its frozen manifest. The same downloader supports `EVA_21M` and +`EVA_1.4B_GLM`. Other directories are available through the fixed file listing; +public availability alone does not establish correspondence to every paper row. + +## OpenRNA training data + +[OpenRNA v1](https://huggingface.co/datasets/GENTEL-Lab/OpenRNA-v1-114M) provides +the released sequence collection. Use its file listing to choose the required +subset and record the downloaded revision and hashes. Dataset download size +varies by subset. Original training splits and every run-to-checkpoint binding +are not reconstructed by this repository's synthetic training example. + +## Deposited analysis data + +[Benchmark and experiment data](https://doi.org/10.5281/zenodo.19027142): + +| Archive | Bytes | Contents | +|---|---:|---| +| `mrna_codon_optimization.tar.gz` | 25,628 | Archived case-study results | +| `generated_rna_sequences.tar.gz` | 162,986,756 | Generated/reference sequences and calculated features | +| `generrna_generation.tar.gz` | 13,487,400 | Generation-comparison data | +| `mrna_6species.tar.gz` | 39,780,876 | Six-species sequence and feature comparisons | + +These data deposits are distinct from the code archive and do not have to be +downloaded for the bundled Milena example. Consult [paper workflows](PAPER_WORKFLOWS.md) +for which figures have a portable command and complete inputs. + +## Runtime image + +Prefer building the current [Docker recipe](INSTALLATION.md#gpu-runtime-docker) +for the repaired code. The legacy [environment deposit](https://doi.org/10.5281/zenodo.18786989) +currently resolves to a record with eight split archive parts, not a single +`eva_latest.tar.gz` download. The retrieved version DOI is +[10.5281/zenodo.22093406](https://doi.org/10.5281/zenodo.22093406). +Its files are not evidence that it contains the latest main-branch repairs. +The checked metadata and advertised checksums are retained in the JSON record. + +## Comparison models and access checks + +[Official comparison-model resources](OFFICIAL_MODEL_RESOURCES.md) links the +upstream code and records unresolved historical bindings. Run the independent +access report from the repository root with Python 3.10 or 3.11: + +```bash +python scripts/check_resource_links.py --output results/resource-access.json +``` + +Exit 0 means the report was written. Inspect each `access` field: network +errors and unavailable revisions are reported without changing CPU regression +status. The tool retrieves metadata only; it neither downloads model bodies +nor replaces missing weights. CPU CI performs offline file/hash checks. + +The [September 9 access report](../reproduction/release/resource_access_20260909.json) +is a dated observation. External services and their current default branches +can change. Each data/model resource retains its upstream license and terms. diff --git a/docs/REVIEWER_REQUIREMENTS.md b/docs/REVIEWER_REQUIREMENTS.md new file mode 100644 index 0000000..005f27d --- /dev/null +++ b/docs/REVIEWER_REQUIREMENTS.md @@ -0,0 +1,28 @@ +# Software requirements and evidence + +This page maps the six software requests in the September 2, 2026 review to +public entry points and validation evidence. It describes release scope, +not editorial acceptance. + +| Requirement | Public entry and evidence | Status and remaining scope | +|---|---|---| +| Benchmark scores for EVA and comparison methods | [Milena workflow](REPRODUCTION.md#gpu-run-the-complete-milena-benchmark), [comparison protocols](BENCHMARK_PROTOCOLS.md), [workflow index](PAPER_WORKFLOWS.md) | EVA representative inference verified; competition-wide reproduction remains partial where original resources or producers are missing. | +| Executable training/evaluation components, or explicit omissions | [Training](../training/pretrain/README.md), [fine-tuning](../finetune/aptamer/script/README.md), [smoke workflow](REPRODUCTION.md#training-and-fine-tuning) | Small training and save/reload verified. Full pretraining, missing dense-model code and unresolved paper configurations are not claimed. | +| Explicit dependency failures | `tests/test_reproducibility.py`; [validation](RELEASE_STATUS.md) | Dependency failures, nonfinite scores and invalid training paths tested. Archived legacy sources are not the supported CLI. | +| Precise checkpoint and dataset identities | `reproduction/milena_14b/manifest.json`, [resources](RESOURCES.md), [comparison resources](OFFICIAL_MODEL_RESOURCES.md) | Representative versions/hashes fixed. Other missing versions remain explicit; formal release and code DOI pending. | +| Public prompt, conditioning and input specifications | [CLI reference](USAGE.md), [Milena protocol](REPRODUCTION.md#gpu-run-the-complete-milena-benchmark), [benchmark protocols](BENCHMARK_PROTOCOLS.md) | Documentation covers supported entry points; omitted components identified separately. | +| One end-to-end benchmark | [Download → inference → metrics → plots](REPRODUCTION.md#gpu-run-the-complete-milena-benchmark), [expected output](../reproduction/milena_14b/expected/README.md) | Completed for 135 Milena samples. The measured 0.00338 difference is documented and accepted for this example; the reference is preserved. | + +## Checking the evidence + +1. Follow [Installation](INSTALLATION.md), then the selected path in + [Reproduction](REPRODUCTION.md). CPU recalculation and GPU inference are distinct. +2. Inspect predictions/reports: IDs, counts, hashes, environment and comparison + results. `--check-only` verifies resources without running inference. +3. Use [Paper workflows](PAPER_WORKFLOWS.md) for method-specific requirements. + Historical source availability is not counted as executed inference. + +The 128-test clean-archive record is tied to its named commit. GitHub Actions +checks current CPU behavior separately. A passing CPU check or representative +example does not establish reproduction of all manuscript experiments. +Formal release/DOI and the submission's code citation remain separate tasks. diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000..c145988 --- /dev/null +++ b/docs/USAGE.md @@ -0,0 +1,453 @@ +# Using EVA + +Complete CLI reference for an installed [GPU runtime](INSTALLATION.md). +For a runnable paper example, see [Reproduction](REPRODUCTION.md). +Commands below use user-supplied input and checkpoint paths. + +## Condition Control + +EVA supports conditioning on **RNA type** and **species/lineage** for both generation (`generate.py`) and scoring (`predict.py`). These conditions can be used independently or combined. + +### RNA Types + +| RNA Type | Description | +|----------|-------------| +| mRNA | Messenger RNA - carries genetic information from DNA to ribosomes | +| tRNA | Transfer RNA - brings amino acids to the ribosome during translation | +| rRNA | Ribosomal RNA - forms the core of the ribosome structure | +| miRNA | MicroRNA - regulates gene expression | +| lncRNA | Long non-coding RNA - various regulatory functions | +| circRNA | Circular RNA - circularized RNA molecules | +| snoRNA | Small nucleolar RNA - modifies other RNAs | +| snRNA | Small nuclear RNA - involved in splicing | +| piRNA | PIWI-interacting RNA - silences transposons | +| sRNA | Small RNA - general category for small RNA molecules | +| viral_RNA | RNA from viruses | + +### Species/Lineage + +Species can be specified in three ways: `--taxid`, `--species`, or `--lineage` (Greengenes format). + +Common species: + +| TaxID | Species | +|-------|---------| +| 9606 | Homo sapiens (Human) | +| 10090 | Mus musculus (Mouse) | +| 10116 | Rattus norvegicus (Rat) | +| 7227 | Drosophila melanogaster (Fruit fly) | +| 6239 | Caenorhabditis elegans (Nematode) | +| 3702 | Arabidopsis thaliana (Plant) | +| 4932 | Saccharomyces cerevisiae (Yeast) | +| 562 | Escherichia coli (Bacteria) | + +
+ +## Generation + +See the paper for the evaluated datasets, metrics and comparisons. + + + + + + +
+ RNA Landscape Modeling Comparison +
RNA landscape modeling comparison + 		+ Species-specific RNA Landscape Modeling Comparison +
Species-specific modeling comparison +
+ +### CLM + +CLM (Causal Language Model) generates RNA sequences autoregressively from left to right. This is the primary generation mode in EVA. + +#### Unconditional Generation + +Generate sequences without any biological constraints: + +```bash +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --num_seqs 1000 \ + --output /output/unconditional.fa +``` + +#### Conditional Generation + +EVA supports conditioning on **RNA type**, **species** (via TaxID, species name, or lineage string), or both. See [Condition Control](#condition-control) for the full list of supported RNA types and species. + +
+ RNA Type Generation +
+ +```bash +# RNA type only +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --rna_type mRNA \ + --num_seqs 1000 \ + --output /output/mrna.fa + +# Species only (via TaxID) +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --taxid 9606 \ + --num_seqs 1000 \ + --output /output/human.fa + +# Both RNA type and species +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --rna_type mRNA \ + --taxid 9606 \ + --num_seqs 1000 \ + --output /output/human_mrna.fa +``` + +Species can also be specified via `--species homo_sapiens` or `--lineage "D__Eukaryota;P__Chordata;..."` in Greengenes format. + +
+ Species Generation +
+ +#### Continuation Mode + +Extend existing sequences in either direction. Use `--split_ratio` (fraction) or `--split_pos` (exact position) to control the split point. + +**Forward** (extend 3' end): + +```bash +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --input /input/partial_seq.fa \ + --direction forward \ + --split_ratio 0.5 \ + --num_seqs 5 \ + --output /output/continuation.fa +``` + +**Reverse** (extend 5' end): + +```bash +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --input /input/partial_seq.fa \ + --direction reverse \ + --split_pos 699 \ + --num_seqs 20 \ + --output /output/reverse_continuation.fa +``` + +Add `--output_details` to include prompt, ground truth, and generated content in the output. + +### GLM + +GLM (General Language Model) performs span infilling — it masks a region within an existing sequence and generates what should fill the gap based on surrounding context. Like CLM, GLM supports both unconditional and conditional generation. + +#### Unconditional Infilling + +Fill in a masked region without any biological constraints: + +```bash +eva-generate \ + --checkpoint /path/to/model \ + --format glm \ + --input /input/sequences.fa \ + --span_ratio 0.1 \ + --num_seqs 5 \ + --output /output/glm_output.fa +``` + +#### Conditional Infilling + +Condition on RNA type and/or species to generate biologically consistent infills: + +```bash +eva-generate \ + --checkpoint /path/to/model \ + --format glm \ + --input /input/sequences.fa \ + --rna_type mRNA \ + --taxid 9606 \ + --span_ratio 0.2 \ + --num_seqs 5 \ + --output /output/glm_conditional.fa +``` + +#### Span Parameters + +| Parameter | Description | Example | +|-----------|-------------|---------| +| `--span_length` | Fixed number of nucleotides to mask | `--span_length 20` | +| `--span_ratio` | Fraction of sequence to mask | `--span_ratio 0.1` | +| `--span_position` | Where to place the span: `random` or specific index | `--span_position 100` | +| `--span_id` | Which span token to use: `random` or 0-49 | `--span_id 0` | + +### Sampling Parameters + +| Parameter | Description | Recommended Range | +|-----------|-------------|------------------| +| `--temperature` | Controls randomness. Lower = more deterministic, higher = more diverse | 0.1 - 1.5 | +| `--top_k` | Only consider the top k most likely nucleotides at each position | 10 - 100 | +| `--top_p` | Nucleus sampling — consider smallest set of nucleotides whose cumulative probability exceeds p | 0.8 - 0.95 | + +Example with all sampling parameters: + +```bash +eva-generate \ + --checkpoint /path/to/model \ + --format clm \ + --temperature 0.8 \ + --top_k 50 \ + --top_p 0.9 \ + --num_seqs 100 \ + --output /output/sampled.fa +``` + +
+ +## Scoring + +EVA provides sequence log-likelihood scores. Fitness comparisons depend on the +assay and scoring protocol; use the [benchmark guide](BENCHMARK_PROTOCOLS.md) +for paper-specific calculations. + + + + + + +
+ Zero-shot mRNA Fitness Prediction + + Zero-shot ncRNA Fitness Prediction +
+ +Evaluate how well a given sequence fits the model's learned distribution by computing its log-likelihood. Higher (less negative) scores indicate more probable sequences. + +### RNA Mode + +Score RNA sequences and compute per-sequence log-likelihood: + +```bash +eva-predict \ + --checkpoint /path/to/model \ + --input /input/sequences.fa \ + --output /output/scores.json +``` + +Supports `--rna_type` and `--taxid` conditioning, same as generation. + +### Protein Mode + +Score protein sequences by reverse-translating them to RNA first: + +```bash +eva-predict \ + --checkpoint /path/to/model \ + --input /input/proteins.fa \ + --output /output/protein_scores.json \ + --mode protein \ + --codon_optimization first +``` + +`--codon_optimization` options: `first` (first codon in table) or `most_frequent` (most common codon for the species). + +
+ +## Directed Evolution + +EVA supports in-silico directed evolution — an iterative optimization pipeline that improves a given RNA sequence through cycles of mutation, LLM-based fitness scoring, and structural stability evaluation. Starting from an existing RNA sequence, EVA applies point mutations at specified positions, scores each mutant by log-likelihood and Minimum Free Energy (MFE via ViennaRNA), and uses simulated annealing with dynamic beam search to balance exploration and exploitation. Over multiple rounds the temperature cools to converge on optimized sequences, and the top N candidates ranked by combined LLM + MFE score are returned. + + + + + + + + + + +
+ circRNA Illustration + + mRNA Optimization Illustration +
+ circRNA Optimization + + mRNA Optimization +
+ +### Usage + +```bash +eva-evolve \ + --checkpoint /path/to/model \ + --input /input/sequence.fa \ + --output /output/evolved.fa \ + --rna_type mRNA \ + --taxid 9606 \ + --iterations 10 \ + --mutations 5 \ + --beam_width 10 \ + --output_count 5 +``` + +### Key Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--iterations` | Number of evolution cycles | 10 | +| `--mutations` | Total mutations per iteration | 2 | +| `--beam_width` | Beam search width — keeps top candidates after each position | 10 | +| `--output_count` | Number of final sequences to output | 5 | +| `--T_init` / `--T_min` | Simulated annealing temperature range | 1.0 → 0.01 | +| `--cooling_rate` | Temperature decay per iteration | 0.95 | +| `--mutate_positions` | Specific positions to mutate (0-based, comma-separated) | Random | +| `--mutate_range` | Range of positions to mutate (e.g., `0-100`) | Entire sequence | + +
+ +## Batch Processing with YAML + +Define multiple tasks in a single YAML config file. The `defaults` section sets shared parameters, which individual tasks can override. + +### Generation Config Example + +```yaml +checkpoint: /path/to/model +output_dir: ./output + +defaults: + temperature: 1.0 + top_k: 50 + max_length: 8192 + batch_size: 1 + +tasks: + - name: unconditional + mode: generation + format: clm + num_seqs: 1000 + + - name: human_mrna + mode: generation + format: clm + rna_type: mRNA + taxid: "9606" + lineage: "D__Eukaryota;P__Chordata;C__Mammalia;O__Primates;F__Hominidae;G__Homo;S__Homo sapiens" + num_seqs: 1000 + + - name: glm_infill + mode: generation + format: glm + input: ./input/seqs.fa + span_ratio: 0.1 + num_seqs: 5 +``` + +### Scoring Config Example + +```yaml +checkpoint: /path/to/model +output_dir: ./scores + +defaults: + batch_size: 128 + +tasks: + - name: score_basic + mode: scoring + input: ./input/seqs.fa + + - name: score_human_mrna + mode: scoring + input: ./input/seqs.fa + rna_type: mRNA + taxid: "9606" + normalize: true + exclude_special_tokens: true + + - name: score_protein + mode: scoring + input: ./input/proteins.fa + scoring_mode: protein + codon_optimization: first +``` + +### Running + +```bash +eva-generate --config config.yaml # Run all tasks +eva-generate --config config.yaml --task name # Run specific task +eva-generate --config config.yaml --device cuda:1 # Override device +``` + +
+ +## Input/Output Formats + +### Input — FASTA + +``` +>sequence_id_1 +AUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCU +>sequence_id_2 +AUGAAAAUGCGGCCGCAUUACGUAAACGGCCGCAAAUGUUUCCGGCAAA +``` + +### Output — Generation (FASTA) + +``` +>unconditional_0 +AUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCUAUGCGCU +``` + +With `--output_details` (GLM / Continuation): + +``` +>test_seq_sample0_forward_split50 +PROMPT: AUGCGCUAUGCGCUAUGCG +GROUND_TRUTH: CU AUGCGCUAUGCG +GENERATED: CU AAUGCGCUAGCG +FULL_SEQ: AUGCGCUAUGCGCUAAUGCGCUAGCG +``` + +### Output — Scoring (JSON) + +```json +{ + "scores": [ + { + "header": "seq1", + "sequence": "AUGGCCGUAGU...", + "length": 67, + "log_likelihood": -1.25 + } + ] +} +``` + +Higher (less negative) `log_likelihood` means a more probable sequence under the selected model and protocol; it is not a universal measure of experimental fitness. + +### Output — Directed Evolution (FASTA) + +Output is in FASTA format with scores in the header: + +``` +>candidate_1_score=-125.4321_best +AUGC... (evolved sequence) +>candidate_2_score=-128.1056 +AUGC... (second best) +``` + +
diff --git a/pyproject.toml b/pyproject.toml index 4ee4de1..1b2deea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ benchmark = ["scipy>=1.11", "huggingface-hub>=0.34,<1"] dev = [ "build>=1.2", "pytest>=8", + "jsonschema>=4.18,<5", ] notebook = [ "ipython>=8.0", diff --git a/reproduction/milena_14b/expected/README.md b/reproduction/milena_14b/expected/README.md index 8b03ae8..9766bf0 100644 --- a/reproduction/milena_14b/expected/README.md +++ b/reproduction/milena_14b/expected/README.md @@ -20,8 +20,11 @@ objective search was performed to fit the reported value. The report's decimal-precision comparison is a diagnostic against the stored CSV, not a demand for scientifically meaningful agreement to 16 decimal places. The manuscript plot does not annotate an individual Milena coefficient. -Rounding to two decimals is descriptive context only; publication remains -pending the requested explanation of the different archived predictions. +Rounding to two decimals is descriptive context only. The observed 0.00338 +difference is accepted for the representative example and is not a merge or +release blocker. Current execution status and optional strict comparison are +documented in [the reproduction guide](../../../docs/REPRODUCTION.md). +The dated JSON reports below preserve their original status strings and measurements. The run used a cached weight file whose SHA256 matches the public revision's LFS object, plus freshly retrieved public config and tokenizer files. diff --git a/reproduction/release/model_manifests/rnabert.json b/reproduction/release/model_manifests/rnabert.json new file mode 100644 index 0000000..56969ae --- /dev/null +++ b/reproduction/release/model_manifests/rnabert.json @@ -0,0 +1,13 @@ +{ + "source": "Recovered original benchmark cache", + "model": "multimolecule/rnabert", + "revision": "7a9b5d5d5a96931ff056828c0968479098506cea", + "files": { + "config.json": "953a8546ebd2bd515d59fc3fdc86f714d90544253d38d787c8c1a938762bad95", + "model.safetensors": "a0619ebdf92a652b38d89973e1f42c03f057a3b05498b7128146e979934e726f", + "special_tokens_map.json": "425c62f51fae9706f499386aec42b284e177da51d86e6ffd1ac744b3822278eb", + "tokenizer_config.json": "441086142f11ee42ac345cfbfaad31bebbfa8ddcffb4319b7614308df4ec2571", + "vocab.txt": "686c991af8379cbb812ada930086ad17f820327a99097eb7857ae38ef2da2572" + }, + "public_availability": "The historical revision is unavailable; this manifest identifies files but does not download them." +} diff --git a/reproduction/release/model_manifests/rnafm.json b/reproduction/release/model_manifests/rnafm.json new file mode 100644 index 0000000..93fb728 --- /dev/null +++ b/reproduction/release/model_manifests/rnafm.json @@ -0,0 +1,13 @@ +{ + "source": "Recovered original benchmark cache", + "model": "multimolecule/rnafm", + "revision": "2a98c38e8219d666ad35d04a689218ab50a36293", + "files": { + "config.json": "028e14a0fb28c088661df8cd08fdff1dee8238e33d46029fb8af8c153dc6524f", + "model.safetensors": "1fccd0adf65340ebf246cb35c1ce7eb68e0830023092a7561cee26ee40261a19", + "special_tokens_map.json": "425c62f51fae9706f499386aec42b284e177da51d86e6ffd1ac744b3822278eb", + "tokenizer_config.json": "0f9810f042f4c3f206378d452e3a6e115a3c11940277a31b97a632e285fbf9e5", + "vocab.txt": "686c991af8379cbb812ada930086ad17f820327a99097eb7857ae38ef2da2572" + }, + "public_availability": "The historical revision is unavailable; this manifest identifies files but does not download them." +} diff --git a/reproduction/release/model_manifests/rnamsm.json b/reproduction/release/model_manifests/rnamsm.json new file mode 100644 index 0000000..60d3b2f --- /dev/null +++ b/reproduction/release/model_manifests/rnamsm.json @@ -0,0 +1,13 @@ +{ + "source": "Recovered original benchmark cache", + "model": "multimolecule/rnamsm", + "revision": "ad61a4915674596557b3c3c4fe4fd7162d36acc8", + "files": { + "config.json": "010b1efbd3e9c8a930e4fa72744876e365ef1b79533e65e41486de92a663e25d", + "model.safetensors": "41f2c8ca7d81cd626b772dbc07de657d67cbf53a4e782ae4e4147f2d254fed24", + "special_tokens_map.json": "425c62f51fae9706f499386aec42b284e177da51d86e6ffd1ac744b3822278eb", + "tokenizer_config.json": "0f9810f042f4c3f206378d452e3a6e115a3c11940277a31b97a632e285fbf9e5", + "vocab.txt": "686c991af8379cbb812ada930086ad17f820327a99097eb7857ae38ef2da2572" + }, + "public_availability": "The historical revision is unavailable; this manifest identifies files but does not download them." +} diff --git a/reproduction/release/public_resource_metadata_20260909.json b/reproduction/release/public_resource_metadata_20260909.json new file mode 100644 index 0000000..c97ddeb --- /dev/null +++ b/reproduction/release/public_resource_metadata_20260909.json @@ -0,0 +1,114 @@ +{ + "checked_date": "2026-09-09", + "model_repository": "GENTEL-Lab/EVA", + "model_revision": "514db6705637c1ec963b728768fc9b34728699ee", + "model_weights": [ + { + "path": "EVA_1.4B_CLM/model_weights.pt", + "bytes": 3057231170, + "sha256": "323c13d571d0be87e450cb7b103bf8b28396e55e877519e2d2fb3c434875d420" + }, + { + "path": "EVA_1.4B_GLM/model_weights.pt", + "bytes": 3057231170, + "sha256": "1ee172fe1bf68c6907590dae630b6e1d8493225dd0b5a34ecc0ead0cc51d4f0c" + }, + { + "path": "EVA_145M/model_weights.pt", + "bytes": 361225818, + "sha256": "785f14ca3db031ab4a70692146cea3d5a192ceb6175271716ae52ab0c7cb5111" + }, + { + "path": "EVA_21M/model_weights.pt", + "bytes": 44860956, + "sha256": "45d56a7399c4936429da149edf5003bbb5999490a64ec7d793bf5ac98116eb01" + }, + { + "path": "EVA_437M/model_weights.pt", + "bytes": 1013942634, + "sha256": "1dec07cef32d888f9f3219cb9fb7c269cb62464e795b4f251ce800d6dde92b81" + } + ], + "zenodo_records": [ + { + "record_id": 19027142, + "title": "EVA: Benchmark and Experiment Data for RNA Generation and Codon Optimization", + "doi": "10.5281/zenodo.19027142", + "files": [ + { + "key": "mrna_codon_optimization.tar.gz", + "size": 25628, + "checksum": "md5:04d0ed6a704250f7916b6fa902264e82" + }, + { + "key": "generated_rna_sequences.tar.gz", + "size": 162986756, + "checksum": "md5:dd722ed031762104b79c9483682e4dbc" + }, + { + "key": "generrna_generation.tar.gz", + "size": 13487400, + "checksum": "md5:bcda4428e44ce22a9deee13dccbc6147" + }, + { + "key": "mrna_6species.tar.gz", + "size": 39780876, + "checksum": "md5:ef70646d594692a55fda6f561ab1f72a" + } + ] + }, + { + "record_id": 18786989, + "title": "EVA1 docker image : A Generative Foundation Model for Universal RNA Modeling and Design", + "doi": "10.5281/zenodo.22093406", + "files": [ + { + "key": "eva_latest.tar.gz.part000", + "size": 1073741824, + "checksum": "md5:e996fdb48d55e2233518feac44c4e413" + }, + { + "key": "eva_latest.tar.gz.part001", + "size": 1073741824, + "checksum": "md5:9658b41dd297c0cc14a34f38fce507fd" + }, + { + "key": "eva_latest.tar.gz.part004", + "size": 1073741824, + "checksum": "md5:f3bc7bd3b5971b9367a76a122bd52475" + }, + { + "key": "eva_latest.tar.gz.part002", + "size": 1073741824, + "checksum": "md5:5d2adef76627ea18cfd605811b1fb829" + }, + { + "key": "eva_latest.tar.gz.part005", + "size": 1073741824, + "checksum": "md5:f198f8d4c31ecb066d55c88d4a17303f" + }, + { + "key": "eva_latest.tar.gz.part003", + "size": 1073741824, + "checksum": "md5:7dfca078d01e815f98c5abfd35f171cd" + }, + { + "key": "eva_latest.tar.gz.part007", + "size": 220584182, + "checksum": "md5:fb71db7fb4cb785f7657a44017a4ed29" + }, + { + "key": "eva_latest.tar.gz.part006", + "size": 1073741824, + "checksum": "md5:f5c01e3a7c6901ebf1aad61b9842f006" + }, + { + "key": "CHECKSUMS.sha256", + "size": 820, + "checksum": "md5:700095c327b1357a5cd11e38a124d86a" + } + ] + } + ], + "scope": "Public resource metadata, not a binding of every artifact to a paper experiment" +} diff --git a/reproduction/release/recovered_competitor_assets_20260909.json b/reproduction/release/recovered_competitor_assets_20260909.json new file mode 100644 index 0000000..8432b98 --- /dev/null +++ b/reproduction/release/recovered_competitor_assets_20260909.json @@ -0,0 +1,121 @@ +{ + "inventory": [ + { + "root": "rna_benchmark/CodonFM/checkpoint", + "exists": true, + "files": [ + { + "path": "CodonFM-1B/NV-CodonFM-Encodon-1B-v1.safetensors", + "bytes": 3645266844, + "sha256": "7880a89baa83156e72d2f3b695c6cd91523067fd86301028bbf7614b2b801607" + }, + { + "path": "CodonFM-1B/config.json", + "bytes": 529, + "sha256": "0aac2cedb1f884882f6723b852dd74b99b4463bbb927cb45822d8a4190d56889" + }, + { + "path": "CodonFM-600M/NV-CodonFM-Encodon-600M-v1.safetensors", + "bytes": 2436161020, + "sha256": "e81e4f63acd5e27be19848024a5912f5416f79192ecd32993fa09cbd2ae2976b" + }, + { + "path": "CodonFM-600M/config.json", + "bytes": 560, + "sha256": "5371b73fc1ca264a7b3567ba8286bacedfaf43bba2ff99e827fd07e7cce6f981" + } + ] + }, + { + "root": "rna_benchmark/biombenchmark/eva_fitness_benchmark/model/ERNIE-RNA", + "exists": true, + "files": [ + { + "path": "environment.yml", + "bytes": 4433, + "sha256": "0c331f17799df6c9e515e6ed3241163f3f827e4c02fbe8f2b15a6e02b9796354" + }, + { + "path": "checkpoint/ERNIE-RNA_checkpoint/ERNIE-RNA_pretrain.pt", + "bytes": 1035221383, + "sha256": "50ac42d4a2af361b8a7d9fb62546e3f380c32893b179554609440339a6676851" + } + ] + }, + { + "root": "biollm_benchmark/model", + "exists": true, + "files": [ + { + "path": "AIdo.RNA/model-weight/config.json", + "bytes": 742, + "sha256": "8ad42f1f4faa962346e40fe1f1203debec53543997de7366afdfaa895168cc4b" + }, + { + "path": "AIdo.RNA/model-weight/pytorch_model-00001-of-00002.safetensors", + "bytes": 4978586096, + "sha256": "bc1b809d40e39d378dae16747b47628ce993f36a0bb83c5ef285c6e3c9b508cd" + }, + { + "path": "AIdo.RNA/model-weight/pytorch_model-00002-of-00002.safetensors", + "bytes": 1467857000, + "sha256": "f62d1a1a5f681f3fd5e68b8b97b2768651e327c6de3b09c475c8825063dc0962" + }, + { + "path": "ERNIE-RNA/environment.yml", + "bytes": 4433, + "sha256": "0c331f17799df6c9e515e6ed3241163f3f827e4c02fbe8f2b15a6e02b9796354" + }, + { + "path": "ERNIE-RNA/checkpoint/ERNIE-RNA_checkpoint/ERNIE-RNA_pretrain.pt", + "bytes": 1035221383, + "sha256": "50ac42d4a2af361b8a7d9fb62546e3f380c32893b179554609440339a6676851" + }, + { + "path": "GenerRNA/model_updated.pt", + "bytes": 3649845020, + "sha256": "4b8350368d27cfa05a5df9e9993aa75ef67f596d476cec1fcb910d16cb3d6859" + }, + { + "path": "GenerRNA/tokenizer/tokenizer_config.json", + "bytes": 394, + "sha256": "2bde25884c7a306d26664cb35813f65c429e6529c5b57dd9be72ec92f7ef1a7b" + } + ] + } + ], + "configuration_metadata": [ + { + "model": "rnafm", + "fields": { + "architectures": [ + "RnaFmForPreTraining" + ], + "model_type": "rnafm", + "transformers_version": "4.50.0" + } + }, + { + "model": "rnabert", + "fields": { + "architectures": [ + "RnaBertForPreTraining" + ], + "model_type": "rnabert", + "transformers_version": "4.50.0" + } + }, + { + "model": "rnamsm", + "fields": { + "architectures": [ + "RnaMsmForPreTraining" + ], + "model_type": "rnamsm", + "transformers_version": "4.50.0" + } + } + ], + "checked_date": "2026-09-09", + "scope": "Read-only file hashes from the recovered historical benchmark mirror; public version binding and full method execution remain separate" +} diff --git a/reproduction/release/recovered_rna_snapshots_20260909.json b/reproduction/release/recovered_rna_snapshots_20260909.json new file mode 100644 index 0000000..8c60249 --- /dev/null +++ b/reproduction/release/recovered_rna_snapshots_20260909.json @@ -0,0 +1,136 @@ +{ + "source": "Recovered original benchmark Hugging Face cache", + "snapshots": [ + { + "repo": "multimolecule/rnafm", + "revision": "2a98c38e8219d666ad35d04a689218ab50a36293", + "files": [ + { + "path": "config.json", + "bytes": 789, + "sha256": "028e14a0fb28c088661df8cd08fdff1dee8238e33d46029fb8af8c153dc6524f" + }, + { + "path": "model.safetensors", + "bytes": 398112460, + "sha256": "1fccd0adf65340ebf246cb35c1ce7eb68e0830023092a7561cee26ee40261a19" + }, + { + "path": "special_tokens_map.json", + "bytes": 224, + "sha256": "425c62f51fae9706f499386aec42b284e177da51d86e6ffd1ac744b3822278eb" + }, + { + "path": "tokenizer_config.json", + "bytes": 1455, + "sha256": "0f9810f042f4c3f206378d452e3a6e115a3c11940277a31b97a632e285fbf9e5" + }, + { + "path": "vocab.txt", + "bytes": 77, + "sha256": "686c991af8379cbb812ada930086ad17f820327a99097eb7857ae38ef2da2572" + } + ], + "current_public_revision": "7d6e73ad3b48e042b378f9a788a56ccb4d573a27", + "current_public_weight_sha256": "ec69333eb894dbefb580e690b0cbe1cbae9845fefd2bc78da914cd14226a2828", + "current_weight_byte_identical": false, + "historical_public_revision_access": "HTTP_404", + "saved_config_metadata": { + "architectures": [ + "RnaFmForPreTraining" + ], + "model_type": "rnafm", + "transformers_version": "4.50.0" + }, + "runtime_binding": "Config records Transformers 4.50.0; exact Multimolecule package/runtime not recovered" + }, + { + "repo": "multimolecule/rnabert", + "revision": "7a9b5d5d5a96931ff056828c0968479098506cea", + "files": [ + { + "path": "config.json", + "bytes": 769, + "sha256": "953a8546ebd2bd515d59fc3fdc86f714d90544253d38d787c8c1a938762bad95" + }, + { + "path": "model.safetensors", + "bytes": 2072000, + "sha256": "a0619ebdf92a652b38d89973e1f42c03f057a3b05498b7128146e979934e726f" + }, + { + "path": "special_tokens_map.json", + "bytes": 224, + "sha256": "425c62f51fae9706f499386aec42b284e177da51d86e6ffd1ac744b3822278eb" + }, + { + "path": "tokenizer_config.json", + "bytes": 1454, + "sha256": "441086142f11ee42ac345cfbfaad31bebbfa8ddcffb4319b7614308df4ec2571" + }, + { + "path": "vocab.txt", + "bytes": 77, + "sha256": "686c991af8379cbb812ada930086ad17f820327a99097eb7857ae38ef2da2572" + } + ], + "current_public_revision": "edd2ce328f3162e7769a911cdb5d835f3e4be740", + "current_public_weight_sha256": "c6a7f596b16d156ac58c9a6ed44cf1b0ae9e586b39beb5694936a962b18f022f", + "current_weight_byte_identical": false, + "historical_public_revision_access": "HTTP_404", + "saved_config_metadata": { + "architectures": [ + "RnaBertForPreTraining" + ], + "model_type": "rnabert", + "transformers_version": "4.50.0" + }, + "runtime_binding": "Config records Transformers 4.50.0; exact Multimolecule package/runtime not recovered" + }, + { + "repo": "multimolecule/rnamsm", + "revision": "ad61a4915674596557b3c3c4fe4fd7162d36acc8", + "files": [ + { + "path": "config.json", + "bytes": 873, + "sha256": "010b1efbd3e9c8a930e4fa72744876e365ef1b79533e65e41486de92a663e25d" + }, + { + "path": "model.safetensors", + "bytes": 383723908, + "sha256": "41f2c8ca7d81cd626b772dbc07de657d67cbf53a4e782ae4e4147f2d254fed24" + }, + { + "path": "special_tokens_map.json", + "bytes": 224, + "sha256": "425c62f51fae9706f499386aec42b284e177da51d86e6ffd1ac744b3822278eb" + }, + { + "path": "tokenizer_config.json", + "bytes": 1455, + "sha256": "0f9810f042f4c3f206378d452e3a6e115a3c11940277a31b97a632e285fbf9e5" + }, + { + "path": "vocab.txt", + "bytes": 77, + "sha256": "686c991af8379cbb812ada930086ad17f820327a99097eb7857ae38ef2da2572" + } + ], + "current_public_revision": "75a68685cf5d0de374651a4797d28a494cab3986", + "current_public_weight_sha256": "d0d6d4438d5327818560496eeaf132be88513ab65cd6584482b5e7caf8f39aa3", + "current_weight_byte_identical": false, + "historical_public_revision_access": "HTTP_404", + "saved_config_metadata": { + "architectures": [ + "RnaMsmForPreTraining" + ], + "model_type": "rnamsm", + "transformers_version": "4.50.0" + }, + "runtime_binding": "Config records Transformers 4.50.0; exact Multimolecule package/runtime not recovered" + } + ], + "checked_date": "2026-09-09", + "scope": "Recovered original cache identities; no new model inference and no rehosting of third-party weights" +} diff --git a/reproduction/release/resource_access_20260909.json b/reproduction/release/resource_access_20260909.json new file mode 100644 index 0000000..c4ded9f --- /dev/null +++ b/reproduction/release/resource_access_20260909.json @@ -0,0 +1,158 @@ +{ + "checked_at": "2026-09-09T06:04:34.128006+00:00", + "checks": [ + { + "id": "eva-checkpoint", + "url": "https://huggingface.co/api/models/GENTEL-Lab/EVA/revision/514db6705637c1ec963b728768fc9b34728699ee", + "purpose": "Pinned EVA checkpoint metadata", + "api_revision": true, + "expected_revision": "514db6705637c1ec963b728768fc9b34728699ee", + "http_status": 200, + "resolved_url": "https://huggingface.co/api/models/GENTEL-Lab/EVA/revision/514db6705637c1ec963b728768fc9b34728699ee", + "access": "AVAILABLE", + "observed_revision": "514db6705637c1ec963b728768fc9b34728699ee" + }, + { + "id": "openrna", + "url": "https://huggingface.co/api/datasets/GENTEL-Lab/OpenRNA-v1-114M", + "purpose": "Public OpenRNA dataset listing; original training split identity is separate", + "api_revision": true, + "http_status": 200, + "resolved_url": "https://huggingface.co/api/datasets/GENTEL-Lab/OpenRNA-v1-114M", + "access": "AVAILABLE", + "observed_revision": "103c79aab4d625828721a25ab98be51763c326bb" + }, + { + "id": "analysis-data", + "url": "https://zenodo.org/api/records/19027142", + "purpose": "Deposited benchmark and analysis data", + "http_status": 200, + "resolved_url": "https://zenodo.org/api/records/19027142", + "access": "AVAILABLE" + }, + { + "id": "legacy-runtime", + "url": "https://zenodo.org/api/records/18786989", + "purpose": "Legacy prebuilt environment; not the current source build", + "http_status": 200, + "resolved_url": "https://zenodo.org/api/records/22093406", + "access": "AVAILABLE" + }, + { + "id": "rnafm-historical", + "url": "https://huggingface.co/api/models/multimolecule/rnafm/revision/2a98c38e8219d666ad35d04a689218ab50a36293", + "purpose": "Historical port revision", + "api_revision": true, + "expected_revision": "2a98c38e8219d666ad35d04a689218ab50a36293", + "http_status": 404, + "access": "UNAVAILABLE" + }, + { + "id": "rnabert-historical", + "url": "https://huggingface.co/api/models/multimolecule/rnabert/revision/7a9b5d5d5a96931ff056828c0968479098506cea", + "purpose": "Historical port revision", + "api_revision": true, + "expected_revision": "7a9b5d5d5a96931ff056828c0968479098506cea", + "http_status": 404, + "access": "UNAVAILABLE" + }, + { + "id": "rnamsm-historical", + "url": "https://huggingface.co/api/models/multimolecule/rnamsm/revision/ad61a4915674596557b3c3c4fe4fd7162d36acc8", + "purpose": "Historical port revision", + "api_revision": true, + "expected_revision": "ad61a4915674596557b3c3c4fe4fd7162d36acc8", + "http_status": 404, + "access": "UNAVAILABLE" + }, + { + "id": "gena-historical", + "url": "https://huggingface.co/api/models/AIRI-Institute/gena-lm-bert-base-t2t-multi/revision/4633e5a1ada905bb7afee6877d71cc12578a95a5", + "purpose": "Historical snapshot; model-name correspondence unresolved", + "api_revision": true, + "expected_revision": "4633e5a1ada905bb7afee6877d71cc12578a95a5", + "http_status": 200, + "resolved_url": "https://huggingface.co/api/models/AIRI-Institute/gena-lm-bert-base-t2t-multi/revision/4633e5a1ada905bb7afee6877d71cc12578a95a5", + "access": "AVAILABLE", + "observed_revision": "4633e5a1ada905bb7afee6877d71cc12578a95a5" + }, + { + "id": "rnafm-upstream", + "url": "https://api.github.com/repos/ml4bio/RNA-FM", + "purpose": "Official source, not a replacement for the historical port", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/ml4bio/RNA-FM", + "access": "AVAILABLE" + }, + { + "id": "ernie", + "url": "https://api.github.com/repos/Bruce-ywj/ERNIE-RNA", + "purpose": "Official source", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/Bruce-ywj/ERNIE-RNA", + "access": "AVAILABLE" + }, + { + "id": "evo2", + "url": "https://api.github.com/repos/arcinstitute/evo2", + "purpose": "Official source", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/arcinstitute/evo2", + "access": "AVAILABLE" + }, + { + "id": "codonfm", + "url": "https://api.github.com/repos/NVIDIA-BioNeMo/CodonFM", + "purpose": "Official source", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/NVIDIA-BioNeMo/CodonFM", + "access": "AVAILABLE" + }, + { + "id": "aido", + "url": "https://api.github.com/repos/genbio-ai/ModelGenerator", + "purpose": "Official framework", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/genbio-ai/ModelGenerator", + "access": "AVAILABLE" + }, + { + "id": "generrna", + "url": "https://huggingface.co/api/models/pfnet/GenerRNA", + "purpose": "Official model listing", + "api_revision": true, + "http_status": 200, + "resolved_url": "https://huggingface.co/api/models/pfnet/GenerRNA", + "access": "AVAILABLE", + "observed_revision": "b2430ee6498d91998e0e37ce20c6bc1766c318a8" + }, + { + "id": "codongpt", + "url": "https://api.github.com/repos/NanilTx/codonGPT_pub", + "purpose": "Paper-linked source", + "http_status": 404, + "access": "UNAVAILABLE" + }, + { + "id": "esm", + "url": "https://api.github.com/repos/facebookresearch/esm", + "purpose": "Official legacy ESM source", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/facebookresearch/esm", + "access": "AVAILABLE" + }, + { + "id": "progen3", + "url": "https://api.github.com/repos/Profluent-AI/progen3", + "purpose": "Official source", + "http_status": 200, + "resolved_url": "https://api.github.com/repos/Profluent-AI/progen3", + "access": "AVAILABLE" + } + ], + "available": 13, + "total": 17, + "weight_downloaded": false, + "inference_executed": false, + "scope": "Resource accessibility; availability alone does not establish paper-model identity." +} diff --git a/reproduction/release/resource_links.json b/reproduction/release/resource_links.json new file mode 100644 index 0000000..34b102b --- /dev/null +++ b/reproduction/release/resource_links.json @@ -0,0 +1,99 @@ +[ + { + "id": "eva-checkpoint", + "url": "https://huggingface.co/api/models/GENTEL-Lab/EVA/revision/514db6705637c1ec963b728768fc9b34728699ee", + "purpose": "Pinned EVA checkpoint metadata", + "api_revision": true, + "expected_revision": "514db6705637c1ec963b728768fc9b34728699ee" + }, + { + "id": "openrna", + "url": "https://huggingface.co/api/datasets/GENTEL-Lab/OpenRNA-v1-114M", + "purpose": "Public OpenRNA dataset listing; original training split identity is separate", + "api_revision": true + }, + { + "id": "analysis-data", + "url": "https://zenodo.org/api/records/19027142", + "purpose": "Deposited benchmark and analysis data" + }, + { + "id": "legacy-runtime", + "url": "https://zenodo.org/api/records/18786989", + "purpose": "Legacy prebuilt environment; not the current source build" + }, + { + "id": "rnafm-historical", + "url": "https://huggingface.co/api/models/multimolecule/rnafm/revision/2a98c38e8219d666ad35d04a689218ab50a36293", + "purpose": "Historical port revision", + "api_revision": true, + "expected_revision": "2a98c38e8219d666ad35d04a689218ab50a36293" + }, + { + "id": "rnabert-historical", + "url": "https://huggingface.co/api/models/multimolecule/rnabert/revision/7a9b5d5d5a96931ff056828c0968479098506cea", + "purpose": "Historical port revision", + "api_revision": true, + "expected_revision": "7a9b5d5d5a96931ff056828c0968479098506cea" + }, + { + "id": "rnamsm-historical", + "url": "https://huggingface.co/api/models/multimolecule/rnamsm/revision/ad61a4915674596557b3c3c4fe4fd7162d36acc8", + "purpose": "Historical port revision", + "api_revision": true, + "expected_revision": "ad61a4915674596557b3c3c4fe4fd7162d36acc8" + }, + { + "id": "gena-historical", + "url": "https://huggingface.co/api/models/AIRI-Institute/gena-lm-bert-base-t2t-multi/revision/4633e5a1ada905bb7afee6877d71cc12578a95a5", + "purpose": "Historical snapshot; model-name correspondence unresolved", + "api_revision": true, + "expected_revision": "4633e5a1ada905bb7afee6877d71cc12578a95a5" + }, + { + "id": "rnafm-upstream", + "url": "https://api.github.com/repos/ml4bio/RNA-FM", + "purpose": "Official source, not a replacement for the historical port" + }, + { + "id": "ernie", + "url": "https://api.github.com/repos/Bruce-ywj/ERNIE-RNA", + "purpose": "Official source" + }, + { + "id": "evo2", + "url": "https://api.github.com/repos/arcinstitute/evo2", + "purpose": "Official source" + }, + { + "id": "codonfm", + "url": "https://api.github.com/repos/NVIDIA-BioNeMo/CodonFM", + "purpose": "Official source" + }, + { + "id": "aido", + "url": "https://api.github.com/repos/genbio-ai/ModelGenerator", + "purpose": "Official framework" + }, + { + "id": "generrna", + "url": "https://huggingface.co/api/models/pfnet/GenerRNA", + "purpose": "Official model listing", + "api_revision": true + }, + { + "id": "codongpt", + "url": "https://api.github.com/repos/NanilTx/codonGPT_pub", + "purpose": "Paper-linked source" + }, + { + "id": "esm", + "url": "https://api.github.com/repos/facebookresearch/esm", + "purpose": "Official legacy ESM source" + }, + { + "id": "progen3", + "url": "https://api.github.com/repos/Profluent-AI/progen3", + "purpose": "Official source" + } +] diff --git a/scripts/check_installed_package.py b/scripts/check_installed_package.py new file mode 100644 index 0000000..dfbb5e8 --- /dev/null +++ b/scripts/check_installed_package.py @@ -0,0 +1,43 @@ +"""Verify wheel imports, package data and installed CLIs outside the source checkout.""" +import argparse +import importlib +import importlib.resources +import json +import os +from pathlib import Path +import shutil +import subprocess + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source-root', type=Path, required=True) + args = parser.parse_args() + source = args.source_root.resolve() + if Path.cwd().resolve().is_relative_to(source): + raise ValueError('Run this check from outside the source checkout') + locations = {} + for name in ('eva', 'tools.generate', 'tools.predict', 'tools.directed_evolution'): + module = importlib.import_module(name) + location = Path(module.__file__).resolve() + if location.is_relative_to(source): + raise ValueError(f'{name} was imported from the source tree: {location}') + locations[name] = str(location) + tokenizer = importlib.resources.files('eva').joinpath('tokenizer.json') + json.loads(tokenizer.read_text()) + data_files = [p.name for p in importlib.resources.files('tools.utils.data').iterdir() if p.name.endswith('.tsv')] + if not data_files: + raise ValueError('The installed wheel is missing lineage TSV data') + env = dict(os.environ) + env.pop('PYTHONPATH', None) + for cli in ('eva-generate', 'eva-predict', 'eva-evolve'): + executable = shutil.which(cli) + if not executable: + raise ValueError('Missing installed CLI: ' + cli) + subprocess.run([executable, '--help'], env=env, check=True, capture_output=True, text=True) + print(json.dumps({'status': 'PASS', 'imports': locations, 'package_data': data_files, + 'cli_help_checks': 3, 'model_inference': False}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/check_repository.py b/scripts/check_repository.py new file mode 100644 index 0000000..6c87728 --- /dev/null +++ b/scripts/check_repository.py @@ -0,0 +1,79 @@ +"""Check maintained documentation, CFF metadata and frozen inputs without network access.""" +from __future__ import annotations + +import html +import json +from pathlib import Path +import re +from urllib.parse import unquote, urlsplit + +ROOT = Path(__file__).resolve().parents[1] + + +def anchors(text): + counts, result = {}, set() + for line in re.sub(r'```.*?```', '', text, flags=re.S).splitlines(): + heading = re.match(r'^#{1,6}\s+(.+?)\s*#*$', line) + if heading: + label = re.sub(r'\[([^]]+)\]\([^)]*\)', r'\1', heading[1]) + slug = re.sub(r'[^\w\- ]', '', label.lower()).replace(' ', '-') + count = counts.get(slug, 0) + counts[slug] = count + 1 + result.add(slug + (f'-{count}' if count else '')) + result.update(re.findall(r'')) + url = urlsplit(target) + if url.scheme or url.netloc: + continue + count += 1 + local = (path.parent / unquote(url.path)).resolve() if url.path else path + if not local.is_relative_to(ROOT) or not local.exists(): + errors.append(f'{path.relative_to(ROOT)}: missing local target {target}') + elif url.fragment and local.suffix == '.md' and unquote(url.fragment) not in anchors(local.read_text()): + errors.append(f'{path.relative_to(ROOT)}: missing section {target}') + if errors: + raise ValueError('\n'.join(errors)) + return count + + +def main(): + import jsonschema + import yaml + + citation = yaml.safe_load((ROOT / 'CITATION.cff').read_text()) + schema = json.loads((ROOT / '.github/citation-file-format-1.2.0.schema.json').read_text()) + jsonschema.Draft7Validator(schema, format_checker=jsonschema.FormatChecker()).validate(citation) + version = re.search(r'__version__\s*=\s*["\']([^"\']+)', (ROOT / 'eva/_version.py').read_text())[1] + if citation.get('version') != version: + raise ValueError('CITATION.cff version does not match the package version') + links = check_links() + # Validate the actual frozen assay and historical-source checksums. + import sys + sys.path.insert(0, str(ROOT)) + from scripts.reproduce_milena import validate_inputs + from scripts.reproduce_historical_benchmark import audit + manifest, records, labels, _, _ = validate_inputs() + report, rows = audit() + if len(records) != manifest['n'] or len(labels) != len(rows) or not report['arithmetic_match']: + raise ValueError('Frozen benchmark input or arithmetic check failed') + print(json.dumps({'status': 'PASS', 'local_links_checked': links, + 'citation_schema': 'CFF 1.2.0', 'version': version, + 'frozen_assay_rows': len(records), 'model_inference': False}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/check_resource_links.py b/scripts/check_resource_links.py new file mode 100644 index 0000000..0e56038 --- /dev/null +++ b/scripts/check_resource_links.py @@ -0,0 +1,61 @@ +"""Report public resource accessibility separately from offline regression checks. + +Exit 0 means the access report was written, not that every URL is available. +This tool never downloads weight bodies or replaces a historical revision. +""" +from __future__ import annotations +import argparse +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +from pathlib import Path +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] + + +def probe(entry): + item = dict(entry) + request = urllib.request.Request(entry['url'], headers={'User-Agent': 'EVA-resource-check/1.0', + 'Accept': 'application/json'}) + try: + with urllib.request.urlopen(request, timeout=20) as response: + item.update(http_status=response.status, resolved_url=response.url, access='AVAILABLE') + if entry.get('api_revision'): + data = json.load(response) + item['observed_revision'] = data.get('sha') + if entry.get('expected_revision') and data.get('sha') != entry['expected_revision']: + item['access'] = 'REVISION_MISMATCH' + except urllib.error.HTTPError as exc: + item.update(http_status=exc.code, access='UNAVAILABLE') + except (OSError, ValueError) as exc: + item.update(access='ACCESS_ERROR', error=f'{type(exc).__name__}: {exc}') + return item + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--manifest', type=Path, default=ROOT / 'reproduction/release/resource_links.json') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + entries = json.loads(args.manifest.read_text()) + if not isinstance(entries, list) or len({x['id'] for x in entries}) != len(entries): + raise ValueError('Resource list must have unique IDs') + if any(not x['url'].startswith('https://') for x in entries): + raise ValueError('Resource links must use HTTPS') + with ThreadPoolExecutor(max_workers=4) as pool: + checks = list(pool.map(probe, entries)) + report = {'checked_at': datetime.now(timezone.utc).isoformat(), 'checks': checks, + 'available': sum(x['access'] == 'AVAILABLE' for x in checks), + 'total': len(checks), 'weight_downloaded': False, 'inference_executed': False, + 'scope': 'Resource accessibility; availability alone does not establish paper-model identity.'} + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open('x') as f: + f.write(json.dumps(report, indent=2) + '\n') + print(json.dumps({'available': report['available'], 'total': report['total'], + 'report': str(args.output)}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/reproduce_milena.py b/scripts/reproduce_milena.py index 08ba412..3edd411 100644 --- a/scripts/reproduce_milena.py +++ b/scripts/reproduce_milena.py @@ -65,6 +65,8 @@ def main(): parser.add_argument('--device', default='cuda:0') parser.add_argument('--memory-limit-gib', type=float, default=5) parser.add_argument('--check-only', action='store_true', help='Verify inputs/checkpoint without inference; never a reproduction pass') + parser.add_argument('--strict-reference', action='store_true', + help='Return 2 after a valid run if its metric differs at the stored reference precision') args = parser.parse_args() if not args.device.startswith('cuda:') or not 0 < args.memory_limit_gib <= 5: parser.error('Select one CUDA device and a memory limit in (0, 5] GiB') @@ -75,7 +77,9 @@ def main(): raise ValueError('Checkpoint files do not match the pinned public release') args.output = args.output.resolve() args.output.mkdir(parents=True, exist_ok=False) - report = {'status':'INPUTS_VERIFIED_ONLY','fresh_inference':False,'n':len(records), + report = {'status':'INPUTS_VERIFIED_ONLY','execution_status':'INPUTS_VERIFIED_ONLY', + 'comparison_status':'NOT_RUN','strict_reference':args.strict_reference, + 'fresh_inference':False,'n':len(records), 'checkpoint':manifest['checkpoint'],'input_sha256':manifest['input_sha256'], 'label_basis':manifest['label_basis'],'protocol':manifest['protocol'], 'paper_result_reproduced':False} @@ -100,7 +104,8 @@ def main(): tolerance = float(Decimal(5).scaleb(Decimal(reference_text).as_tuple().exponent-1)) matched = abs(rho-reference) <= tolerance inference = json.loads((args.output/'inference/report.json').read_text()) - report.update(status='METRIC_MATCH' if matched else 'METRIC_DIFFERENCE_REQUIRES_EXPLANATION', + report.update(status='METRIC_MATCH' if matched else 'METRIC_DIFFERENCE', + comparison_status='MATCH' if matched else 'DIFFERENCE', fresh_inference=True,spearman=rho,reference=reference, absolute_difference=abs(rho-reference),reference_decimal=reference_text, comparison_tolerance=tolerance,matches_reference_precision=matched, @@ -109,7 +114,7 @@ def main(): prediction_rank_correlation=spearman(scores,archive), archive_vector_exact=inference['exact_equal_count']==len(records), paper_result_reproduced=False, - interpretation='Numerical comparison only; release acceptance also requires explaining material differences and confirming paper protocol correspondence.', + interpretation='Execution and reference comparison are reported separately. A successful run does not establish reproduction of every paper result. The documented Milena difference is retained as a numerical observation.', inference_seconds=inference['scoring_seconds'], peak_gpu_allocated_bytes=inference['cuda_peak_allocated_bytes'], versions=inference['environment'],gpu=inference['gpu']) @@ -120,15 +125,16 @@ def main(): subprocess.run([sys.executable,str(ROOT/'scripts/plot_dms_predictions.py'), '--input',str(args.output/'predictions.csv'), '--output-directory',str(args.output/'plot')],check=True) + report['execution_status'] = 'COMPLETED' except Exception as exc: - report.update(status='FAILED',error=f'{type(exc).__name__}: {exc}') + report.update(status='FAILED',execution_status='FAILED',error=f'{type(exc).__name__}: {exc}') raise finally: report['elapsed_seconds']=time.monotonic()-start report['runner_sha256']=sha256(Path(__file__)) dest.write_text(json.dumps(report,indent=2,allow_nan=False)+'\n') print(json.dumps(report,indent=2)) - return 0 if report['matches_reference_precision'] else 2 + return 2 if args.strict_reference and not report['matches_reference_precision'] else 0 if __name__ == '__main__': diff --git a/tests/test_milena_entrypoint.py b/tests/test_milena_entrypoint.py index f7da835..2938bf5 100644 --- a/tests/test_milena_entrypoint.py +++ b/tests/test_milena_entrypoint.py @@ -3,6 +3,11 @@ from pathlib import Path import tempfile import unittest +import hashlib +import subprocess +import sys +import pytest +from scripts import reproduce_milena as runner from scripts.reproduce_milena import validate_inputs, align_fresh, MANIFEST, ROOT class MilenaEntrypointTests(unittest.TestCase): @@ -31,3 +36,106 @@ def test_predictions_align_by_id_and_sequence(self): for data in ['a,AU,1\na,AU,2\n','a,AU,1\n','a,AC,1\nb,CG,2\n','a,AU,nan\nb,CG,2\n']: path.write_text('variant_id,sequence,fresh_score\n'+data) with self.assertRaises(ValueError): align_fresh(records,path) + + +@pytest.fixture +def run_fixture(tmp_path, monkeypatch): + """Exercise CLI orchestration with a deterministic backend, without GPU weights.""" + records = [{'id': k, 'sequence': s} for k, s in [('a', 'AU'), ('b', 'CG'), ('c', 'AC')]] + checkpoint = tmp_path / 'checkpoint' + checkpoint.mkdir() + (checkpoint / 'model_weights.pt').write_bytes(b'test checkpoint, not a model') + manifest = {'checkpoint': {'sha256': {'model_weights.pt': hashlib.sha256( + (checkpoint / 'model_weights.pt').read_bytes()).hexdigest()}}, + 'input_sha256': {}, 'label_basis': 'synthetic test fixture', + 'protocol': {}, 'archive': 'unused-test-archive'} + monkeypatch.setattr(runner, 'validate_inputs', lambda: (manifest, records, [1, 2, 3], [1, 2, 3], '1.000')) + state = {'scores': [1, 3, 2], 'fail': None, 'calls': []} + + def backend(command, **kwargs): + name = Path(command[1]).name + state['calls'].append(name) + if state['fail'] == name: + raise subprocess.CalledProcessError(1, command) + if name == 'run_historical_14b.py': + out = Path(command[command.index('--output') + 1]) + out.mkdir() + with (out / 'predictions.csv').open('w') as f: + writer = csv.writer(f) + writer.writerow(['variant_id', 'sequence', 'fresh_score']) + writer.writerows((r['id'], r['sequence'], score) for r, score in zip(records, state['scores'])) + (out / 'report.json').write_text(json.dumps({'exact_equal_count': 0, + 'scoring_seconds': 0.1, 'cuda_peak_allocated_bytes': 0, + 'environment': {}, 'gpu': 'synthetic backend'})) + elif name == 'plot_dms_predictions.py': + out = Path(command[command.index('--output-directory') + 1]) + out.mkdir() + (out / 'diagnostic.svg').write_text('') + else: + raise AssertionError('Unexpected execution path: ' + name) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(runner.subprocess, 'run', backend) + + def invoke(*flags): + output = tmp_path / 'output' + monkeypatch.setattr(sys, 'argv', ['reproduce_milena.py', '--checkpoint', str(checkpoint), + '--output', str(output), *flags]) + return runner.main(), json.loads((output / 'report.json').read_text()) + + return invoke, state, tmp_path + + +@pytest.mark.parametrize('strict,match,expected', [(False, False, 0), (True, False, 2), + (False, True, 0), (True, True, 0)]) +def test_execution_success_is_separate_from_reference_comparison(run_fixture, strict, match, expected): + invoke, state, root = run_fixture + if match: + state['scores'] = [1, 2, 3] + code, report = invoke(*(['--strict-reference'] if strict else [])) + assert code == expected + assert report['execution_status'] == 'COMPLETED' + assert report['comparison_status'] == ('MATCH' if match else 'DIFFERENCE') + assert report['matches_reference_precision'] == match + assert report['absolute_difference'] == (0 if match else 0.5) + assert report['fresh_inference'] and not report['paper_result_reproduced'] + assert (root / 'output/predictions.csv').is_file() + assert state['calls'] == ['run_historical_14b.py', 'plot_dms_predictions.py'] + + +def test_check_only_never_claims_inference(run_fixture): + invoke, state, _ = run_fixture + code, report = invoke('--check-only', '--strict-reference') + assert code == 0 and not state['calls'] + assert report['execution_status'] == 'INPUTS_VERIFIED_ONLY' + assert report['comparison_status'] == 'NOT_RUN' + assert not report['fresh_inference'] + + +@pytest.mark.parametrize('failure', ['run_historical_14b.py', 'plot_dms_predictions.py']) +def test_backend_or_plot_failure_preserves_failure_report(run_fixture, failure): + invoke, state, root = run_fixture + state['fail'] = failure + with pytest.raises(subprocess.CalledProcessError): + invoke() + report = json.loads((root / 'output/report.json').read_text()) + assert report['status'] == report['execution_status'] == 'FAILED' + assert 'error' in report + + +@pytest.mark.parametrize('scores', [[1, 2], [1, float('nan'), 3], [2, 2, 2]]) +def test_invalid_backend_output_is_not_a_success(run_fixture, scores): + invoke, state, root = run_fixture + state['scores'] = scores + with pytest.raises(ValueError): + invoke() + assert json.loads((root / 'output/report.json').read_text())['execution_status'] == 'FAILED' + assert state['calls'] == ['run_historical_14b.py'] + + +def test_changed_checkpoint_fails_before_backend_launch(run_fixture): + invoke, state, root = run_fixture + (root / 'checkpoint/model_weights.pt').write_bytes(b'changed') + with pytest.raises(ValueError, match='Checkpoint files do not match'): + invoke() + assert not state['calls'] From 03643c70f344baf676d0055fbb80fdf3a3e14e16 Mon Sep 17 00:00:00 2001 From: Yanjie Huang Date: Tue, 8 Sep 2026 23:25:04 -0700 Subject: [PATCH 2/2] Defer fine-tuning GPU import and record both Milena exit modes --- docs/INSTALLATION.md | 1 + docs/RELEASE_STATUS.md | 20 +++++- docs/REVIEWER_REQUIREMENTS.md | 6 +- finetune/train_finetune.py | 5 +- .../release/usability_gpu_20260909/README.md | 18 ++++++ .../default_report.json | 63 +++++++++++++++++++ .../usability_gpu_20260909/strict_report.json | 63 +++++++++++++++++++ .../usability_gpu_20260909/summary.json | 39 ++++++++++++ 8 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 reproduction/release/usability_gpu_20260909/README.md create mode 100644 reproduction/release/usability_gpu_20260909/default_report.json create mode 100644 reproduction/release/usability_gpu_20260909/strict_report.json create mode 100644 reproduction/release/usability_gpu_20260909/summary.json diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 220314b..8a7c1c3 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -76,6 +76,7 @@ file on any machine that has Docker: ```bash docker build -f docker/Dockerfile -t eva:local . +docker tag eva:local eva:latest # docker/EVA.def expects this local tag singularity build eva_latest.sif docker/EVA.def ``` diff --git a/docs/RELEASE_STATUS.md b/docs/RELEASE_STATUS.md index c4ad41e..0dedf4e 100644 --- a/docs/RELEASE_STATUS.md +++ b/docs/RELEASE_STATUS.md @@ -5,7 +5,25 @@ repairs were merged at `9b5d25ac7cb26de88420f56021f4c0eeb2052e9e`. A formal release for this revision and an archival code DOI are pending; the older `v1.1.1` release does not contain these repairs. -## Recorded validation: September 9, 2026 +## Usability update validation: September 9, 2026 + +The updated Milena entry point was run twice on an idle A100-SXM4-80GB, +with the existing pinned runtime, one visible GPU and batch size 1. Each run +scored all 135 inputs, produced metrics and plots, and exactly matched the +saved fresh-prediction vector. Default mode returned **0**; the optional +strict-reference mode returned **2**. Scoring took about 12 seconds per run; +peak PyTorch-allocated GPU memory was 2.94 GiB. +The [dated reports](../reproduction/release/usability_gpu_20260909/README.md) +record the source commit, file hashes, environment and both execution modes. + +The expanded suite passed **139 tests** without skips in the existing Docker +runtime. Current Python 3.10/3.11 CPU jobs, source-archive execution, wheel +installation outside the checkout and CLI checks are available in +[GitHub Actions](https://github.com/GENTEL-lab/EVA/actions/workflows/cpu.yml). +The clean CPU environment also exposed an eager GPU-backend import in the +fine-tuning entry point; that import now occurs only during model construction. + +## Earlier clean source-build validation: September 9, 2026 The source archive at `70c6c4f7aff2cf12210d8b280c1d4569ca22c0a0` was built in a clean Docker environment. The subsequent `9b5d25a` commit added documentation diff --git a/docs/REVIEWER_REQUIREMENTS.md b/docs/REVIEWER_REQUIREMENTS.md index 005f27d..47cae83 100644 --- a/docs/REVIEWER_REQUIREMENTS.md +++ b/docs/REVIEWER_REQUIREMENTS.md @@ -22,7 +22,9 @@ not editorial acceptance. 3. Use [Paper workflows](PAPER_WORKFLOWS.md) for method-specific requirements. Historical source availability is not counted as executed inference. -The 128-test clean-archive record is tied to its named commit. GitHub Actions -checks current CPU behavior separately. A passing CPU check or representative +The 128-test clean-archive record and the later 139-test/container and two-mode +GPU records are described in [validation status](RELEASE_STATUS.md), with their +respective scope and source identities. GitHub Actions checks current CPU behavior +separately. A passing CPU check or representative example does not establish reproduction of all manuscript experiments. Formal release/DOI and the submission's code citation remain separate tasks. diff --git a/finetune/train_finetune.py b/finetune/train_finetune.py index 2142123..a6e5b48 100644 --- a/finetune/train_finetune.py +++ b/finetune/train_finetune.py @@ -38,7 +38,6 @@ project_root = Path(__file__).parent.parent sys.path.insert(0, str(project_root)) -from eva.causal_lm import create_eva_model from eva.config import EvaConfig from eva.lineage_tokenizer import LineageRNATokenizer @@ -333,6 +332,10 @@ def _setup_model(self): checkpoint = self.training_config.get('resume_from_pretrain') if not checkpoint: raise ValueError('Finetuning requires training_config.resume_from_pretrain') + # Import the compiled GPU backend only when constructing a model. + # Configuration checks and CLI help must remain usable on CPU hosts. + from eva.causal_lm import create_eva_model + tokenizer_dir = Path(checkpoint) if Path(checkpoint).is_dir() else Path(checkpoint).parent self.tokenizer = LineageRNATokenizer.from_pretrained(str(tokenizer_dir)) diff --git a/reproduction/release/usability_gpu_20260909/README.md b/reproduction/release/usability_gpu_20260909/README.md new file mode 100644 index 0000000..0a84308 --- /dev/null +++ b/reproduction/release/usability_gpu_20260909/README.md @@ -0,0 +1,18 @@ +# Milena exit-mode validation — September 9, 2026 + +The [summary](summary.json) binds the source commit, runner, protocol manifest +and runtime image. Both runs used one idle NVIDIA A100-SXM4-80GB, batch size 1 +and the pinned public EVA 1.4B CLM checkpoint. All 135 fresh predictions in each +run exactly equaled the previously saved new-inference vector in +[expected/predictions.csv](../../milena_14b/expected/predictions.csv). + +- [Default report](default_report.json): complete inference, metric calculation + and plots; exit code **0**. +- [Strict report](strict_report.json): the same successful execution with + `--strict-reference`; exit code **2** for the retained reference difference. + +Spearman remains 0.8394237924835843 versus 0.8360456283218484. These records +validate the new execution/comparison distinction; they do not claim that +all paper experiments have been reproduced. Reports retain the original +comparison fields. Full-process elapsed time in the summary also includes +Python startup; `inference_seconds` in each report measures scoring only. diff --git a/reproduction/release/usability_gpu_20260909/default_report.json b/reproduction/release/usability_gpu_20260909/default_report.json new file mode 100644 index 0000000..6e1f9d0 --- /dev/null +++ b/reproduction/release/usability_gpu_20260909/default_report.json @@ -0,0 +1,63 @@ +{ + "status": "METRIC_DIFFERENCE", + "execution_status": "COMPLETED", + "comparison_status": "DIFFERENCE", + "strict_reference": false, + "fresh_inference": true, + "n": 135, + "checkpoint": { + "repo": "GENTEL-Lab/EVA", + "revision": "514db6705637c1ec963b728768fc9b34728699ee", + "subdirectory": "EVA_1.4B_CLM", + "sha256": { + "model_weights.pt": "323c13d571d0be87e450cb7b103bf8b28396e55e877519e2d2fb3c434875d420", + "config.json": "a8571fcd6b88c3d4667d764d657e590cc589fc1316821bdb850d901380b15c47", + "tokenizer.json": "9315168d44e3d6df4cdd0c48f3ab6d5111beb13ac8b163a380c59a6fac5a021d" + } + }, + "input_sha256": { + "notebooks/prediction/data/ncRNA/fasta/Milena_2021_cata.fasta": "587195ca555dff13de482e88df5509065d55788dc886a285ca07a8b27f37a227", + "notebooks/prediction/data/ncRNA/label/Milena_2021_cata_intensities.json": "86a724affa00904e54f5d2532607fdc8d3fb1560ba385e53e8f242ab78efe9a2", + "notebooks/prediction/data/ncRNA_13datasets_spearman.csv": "8acfc40534b60e725aac309ba4b8990d0660b266f6ae8183f42e8e63e1334d14", + "reproduction/benchmark/fixtures/milena/eva_1_4b_wotag_scores.json": "8d5b47937e431acbe3684884291d5b9372a67b3d3461f5ba5c4eb9085af478c1" + }, + "label_basis": "Current local submission data, confirmed by the author on 2026-09-09; positional pairing frozen by FASTA and label hashes; historical header metadata is not used as ground truth.", + "protocol": { + "source": "reproduction/benchmark/historical_14b", + "prompt": "5{RNA}3", + "conditioning": null, + "reduce": "sum", + "excluded_targets": [ + "5", + "3", + "" + ], + "dtype": "bfloat16", + "batch_size": 1, + "moe_world_size": 1 + }, + "paper_result_reproduced": false, + "spearman": 0.8394237924835843, + "reference": 0.8360456283218484, + "absolute_difference": 0.0033781641617359748, + "reference_decimal": "0.8360456283218484", + "comparison_tolerance": 5e-17, + "matches_reference_precision": false, + "comparison_basis": "Stored reference CSV decimal precision; not a scientific acceptance tolerance", + "archive_spearman": 0.8360456283218484, + "prediction_rank_correlation": 0.977465613110916, + "archive_vector_exact": false, + "interpretation": "Execution and reference comparison are reported separately. A successful run does not establish reproduction of every paper result. The documented Milena difference is retained as a numerical observation.", + "inference_seconds": 12.460099731004448, + "peak_gpu_allocated_bytes": 3157299200, + "versions": { + "torch": "2.5.1+cu124", + "transformers": "4.55.0", + "tokenizers": "0.21.4", + "megablocks": "0.7.0", + "flash-attn": null + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "elapsed_seconds": 23.690791673012427, + "runner_sha256": "7fb8bb39dcd8e29a2a94595064f77fbdd805a1819f24534e3289576f66b6b93b" +} diff --git a/reproduction/release/usability_gpu_20260909/strict_report.json b/reproduction/release/usability_gpu_20260909/strict_report.json new file mode 100644 index 0000000..17af2ce --- /dev/null +++ b/reproduction/release/usability_gpu_20260909/strict_report.json @@ -0,0 +1,63 @@ +{ + "status": "METRIC_DIFFERENCE", + "execution_status": "COMPLETED", + "comparison_status": "DIFFERENCE", + "strict_reference": true, + "fresh_inference": true, + "n": 135, + "checkpoint": { + "repo": "GENTEL-Lab/EVA", + "revision": "514db6705637c1ec963b728768fc9b34728699ee", + "subdirectory": "EVA_1.4B_CLM", + "sha256": { + "model_weights.pt": "323c13d571d0be87e450cb7b103bf8b28396e55e877519e2d2fb3c434875d420", + "config.json": "a8571fcd6b88c3d4667d764d657e590cc589fc1316821bdb850d901380b15c47", + "tokenizer.json": "9315168d44e3d6df4cdd0c48f3ab6d5111beb13ac8b163a380c59a6fac5a021d" + } + }, + "input_sha256": { + "notebooks/prediction/data/ncRNA/fasta/Milena_2021_cata.fasta": "587195ca555dff13de482e88df5509065d55788dc886a285ca07a8b27f37a227", + "notebooks/prediction/data/ncRNA/label/Milena_2021_cata_intensities.json": "86a724affa00904e54f5d2532607fdc8d3fb1560ba385e53e8f242ab78efe9a2", + "notebooks/prediction/data/ncRNA_13datasets_spearman.csv": "8acfc40534b60e725aac309ba4b8990d0660b266f6ae8183f42e8e63e1334d14", + "reproduction/benchmark/fixtures/milena/eva_1_4b_wotag_scores.json": "8d5b47937e431acbe3684884291d5b9372a67b3d3461f5ba5c4eb9085af478c1" + }, + "label_basis": "Current local submission data, confirmed by the author on 2026-09-09; positional pairing frozen by FASTA and label hashes; historical header metadata is not used as ground truth.", + "protocol": { + "source": "reproduction/benchmark/historical_14b", + "prompt": "5{RNA}3", + "conditioning": null, + "reduce": "sum", + "excluded_targets": [ + "5", + "3", + "" + ], + "dtype": "bfloat16", + "batch_size": 1, + "moe_world_size": 1 + }, + "paper_result_reproduced": false, + "spearman": 0.8394237924835843, + "reference": 0.8360456283218484, + "absolute_difference": 0.0033781641617359748, + "reference_decimal": "0.8360456283218484", + "comparison_tolerance": 5e-17, + "matches_reference_precision": false, + "comparison_basis": "Stored reference CSV decimal precision; not a scientific acceptance tolerance", + "archive_spearman": 0.8360456283218484, + "prediction_rank_correlation": 0.977465613110916, + "archive_vector_exact": false, + "interpretation": "Execution and reference comparison are reported separately. A successful run does not establish reproduction of every paper result. The documented Milena difference is retained as a numerical observation.", + "inference_seconds": 11.452969689999009, + "peak_gpu_allocated_bytes": 3157299200, + "versions": { + "torch": "2.5.1+cu124", + "transformers": "4.55.0", + "tokenizers": "0.21.4", + "megablocks": "0.7.0", + "flash-attn": null + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "elapsed_seconds": 22.362902776992996, + "runner_sha256": "7fb8bb39dcd8e29a2a94595064f77fbdd805a1819f24534e3289576f66b6b93b" +} diff --git a/reproduction/release/usability_gpu_20260909/summary.json b/reproduction/release/usability_gpu_20260909/summary.json new file mode 100644 index 0000000..c97fa7d --- /dev/null +++ b/reproduction/release/usability_gpu_20260909/summary.json @@ -0,0 +1,39 @@ +{ + "source_commit": "d55eecd6931ed3386b59911f9a67dde512af7fe5", + "source_files_verified": 744, + "source_sha256": { + "scripts/reproduce_milena.py": "7fb8bb39dcd8e29a2a94595064f77fbdd805a1819f24534e3289576f66b6b93b", + "reproduction/benchmark/run_historical_14b.py": "e9b9b7bae229acd4dc2ce118eb1c0a0ef051dcfc4618877630e4714468300d91", + "reproduction/milena_14b/manifest.json": "bf2c3722f4e7ac201e9e9212cc9b0ed5213ef68cab0924889c4f8635fe531b8f" + }, + "runs": [ + { + "mode": "default", + "exit_code": 0, + "n": 135, + "spearman": 0.8394237924835843, + "absolute_difference": 0.0033781641617359748, + "execution_status": "COMPLETED", + "comparison_status": "DIFFERENCE", + "predictions_equal_saved": true, + "elapsed_seconds": 26.825981240006513, + "peak_gpu_allocated_bytes": 3157299200 + }, + { + "mode": "strict", + "exit_code": 2, + "n": 135, + "spearman": 0.8394237924835843, + "absolute_difference": 0.0033781641617359748, + "execution_status": "COMPLETED", + "comparison_status": "DIFFERENCE", + "predictions_equal_saved": true, + "elapsed_seconds": 25.091115621995414, + "peak_gpu_allocated_bytes": 3157299200 + } + ], + "validation_date_utc": "2026-09-09", + "runtime_image_id": "sha256:326ba2c34619dc6f3d2703914ed85b009f8da60b3cf4813e2a1e74da85cc8ec5", + "gpu": "NVIDIA A100-SXM4-80GB", + "scope": "Two independent Milena runs, one visible GPU, batch size 1. Checkpoint and scoring code unchanged; not full-paper reproduction." +}