diff --git a/data/normalization/SOFTNAME/amber/codemeta.json b/data/normalization/SOFTNAME/amber/codemeta.json
new file mode 100644
index 0000000..1338c50
--- /dev/null
+++ b/data/normalization/SOFTNAME/amber/codemeta.json
@@ -0,0 +1,115 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "https://orcid.org/0000-0002-2574-6066",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Vertex Pharmaceuticals, 40 Allston Street, Cambridge, MA 02138, USA"
+ },
+ "familyName": "Pearlman",
+ "givenName": "David A. "
+ },
+ {
+ "id": "_:author_2",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Molecular Biology and Chemistry, Scripps Research Institute, La Jolla, CA 92037, USA"
+ },
+ "familyName": "Case",
+ "givenName": "David A. "
+ },
+ {
+ "id": "_:author_3",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Pharmaceutical Chemistry, University of California, San Francisco, CA 94143, USA"
+ },
+ "familyName": "Caldwell",
+ "givenName": "James W. "
+ },
+ {
+ "id": "_:author_4",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Pharmaceutical Chemistry, University of California, San Francisco, CA 94143, USA"
+ },
+ "familyName": "Ross",
+ "givenName": "Wilson S. "
+ },
+ {
+ "id": "https://orcid.org/0000-0003-0298-3904",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Pharmaceutical Chemistry, University of California, San Francisco, CA 94143, USA"
+ },
+ "familyName": "Cheatham III",
+ "givenName": "Thomas E. "
+ },
+ {
+ "id": "_:author_6",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Molecular Biology and Chemistry, Scripps Research Institute, La Jolla, CA 92037, USA"
+ },
+ "familyName": "DeBolt",
+ "givenName": " Steve "
+ },
+ {
+ "id": "https://orcid.org/0000-0001-6294-8057",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": " Department of Medicinal Chemistry, University of Minnesota, Minneapolis, MN 9055455, USA"
+ },
+ "familyName": "Ferguson",
+ "givenName": "David "
+ },
+ {
+ "id": "_:author_8",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Smith Kline Beecham Pharmaceuticals, 709 Swedeland Road, King of Prussia, PA 19406, USA"
+ },
+ "familyName": "Seibel",
+ "givenName": "George "
+ },
+ {
+ "id": "_:author_9",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Pharmaceutical Chemistry, University of California, San Francisco, CA 94143, USA"
+ },
+ "familyName": "Kollman",
+ "givenName": " Peter "
+ }
+ ],
+ "codeRepository": "https://github.com/Amber-MD/AmberClassic.git",
+ "dateModified": "2024-12-21",
+ "description": "Amber is a suite of biomolecular simulation programs. It began in the late 1970's, and is maintained by an active development community; see our history page and our contributors page for more information.\nThe term \"Amber\" refers to two things. First, it is a set of molecular mechanical force fields for the simulation of biomolecules; these force fields are in the public domain, and are used in a variety of simulation programs. Second, it is a package of molecular simulation programs which includes source code and demos.\nAmber is distributed in two parts: AmberTools25 and Amber24. ",
+ "downloadUrl": "https://github.com/Amber-MD/AmberClassic/archive/refs/tags/v1.0.tar.gz",
+ "keywords": "molecular-dynamics",
+ "license": "https://spdx.org/licenses/GPL-2.0",
+ "name": " amber",
+ "programmingLanguage": [
+ "C++",
+ "C",
+ "Fortran",
+ "Makefile",
+ "HTML",
+ "Shell"
+ ],
+ "relatedLink": "https://ambermd.org/",
+ "version": "1.0",
+ "referencePublication": "https://doi.org/10.1016/0010-4655(95)00041-D"
+}
diff --git a/data/normalization/SOFTNAME/charmm-gui/codemeta.json b/data/normalization/SOFTNAME/charmm-gui/codemeta.json
new file mode 100644
index 0000000..34360e1
--- /dev/null
+++ b/data/normalization/SOFTNAME/charmm-gui/codemeta.json
@@ -0,0 +1,54 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "https://orcid.org/0000-0002-4104-6473",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Chemistry, The University of Kansas, 2030 Becker Drive, Lawrence, Kansas 66047"
+ },
+ "familyName": "Jo",
+ "givenName": "Sunhwan "
+ },
+ {
+ "id": "_:author_2",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Molecular Biosciences and Center for Bioinformatics, The University of Kansas, 2030 Becker Drive, Lawrence, Kansas 66047"
+ },
+ "familyName": "Kim",
+ "givenName": "Taehoon "
+ },
+ {
+ "id": "_:author_3",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Chemistry, The University of Kansas, 2030 Becker Drive, Lawrence, Kansas 66047"
+ },
+ "familyName": "Iyer",
+ "givenName": "Vidyashankara G. "
+ },
+ {
+ "id": "_:author_4",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Molecular Biosciences and Center for Bioinformatics, The University of Kansas, 2030 Becker Drive, Lawrence, Kansas 66047"
+ },
+ "familyName": "Im",
+ "givenName": "Wonpil "
+ }
+ ],
+ "codeRepository": "https://github.com/charmm-gui",
+ "dateModified": "2022-07-00",
+ "description": "Since its original development in 2006, CHARMM-GUI has proven to be an ideal web-based platform to interactively build complex systems and prepare their inputs with well-established and reproducible simulation protocols for state-of-the-art molecular simulations using widely used simulation packages such as CHARMM, NAMD, GROMACS, AMBER, GENESIS, Tinker, LAMMPS, Desmond, and OpenMM. The CHARMM-GUI development project has been widely adopted for various purposes and now contains a number of different modules designed to set up a broad range of molecular simulation systems in Input Generator. Many original modules were developed as an in-house effort, but we have established close collaborations with the developers of CHARMM and other MD simulation packages for addition of newer modules.",
+ "name": "charmm-gui",
+ "relatedLink": "https://www.charmm-gui.org/",
+ "version": "Version 3.8",
+ "referencePublication": "https://doi.org/10.1002/jcc.20945"
+}
diff --git a/data/normalization/SOFTNAME/espresso++/codemeta.json b/data/normalization/SOFTNAME/espresso++/codemeta.json
new file mode 100644
index 0000000..db36b93
--- /dev/null
+++ b/data/normalization/SOFTNAME/espresso++/codemeta.json
@@ -0,0 +1,124 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "_:author_1",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Max Planck Institute for Polymer Research, Ackermannweg 10, 55128 Mainz, Germany"
+ },
+ "familyName": "Halverson",
+ "givenName": "Jonathan D."
+ },
+ {
+ "id": "_:author_2",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Fraunhofer Institute for Algorithms and Scientific Computing, Schloss Birlinghoven, 53754 Sankt Augustin, Germany"
+ },
+ "familyName": "Brandes",
+ "givenName": "Thomas"
+ },
+ {
+ "id": "https://orcid.org/0000-0002-0189-7935",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Fraunhofer Institute for Algorithms and Scientific Computing, Schloss Birlinghoven, 53754 Sankt Augustin, Germany"
+ },
+ "familyName": "Lenz",
+ "givenName": "Olaf"
+ },
+ {
+ "id": "_:author_4",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Fraunhofer Institute for Algorithms and Scientific Computing, Schloss Birlinghoven, 53754 Sankt Augustin, Germany"
+ },
+ "familyName": "Arnold",
+ "givenName": "Axel "
+ },
+ {
+ "id": "https://orcid.org/0000-0003-2313-7379",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Max Planck Institute for Polymer Research, Ackermannweg 10, 55128 Mainz, Germany"
+ },
+ "familyName": "Starchenko",
+ "givenName": " Vitaliy "
+ },
+ {
+ "id": "_:author_6",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Max Planck Institute for Polymer Research, Ackermannweg 10, 55128 Mainz, Germany"
+ },
+ "familyName": "Kremer",
+ "givenName": "Kurt"
+ },
+ {
+ "id": "_:author_7",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "National Institute of Chemistry, Hajdrihova 19, 1000 Ljubljana, Slovenia"
+ },
+ "familyName": "Bevc",
+ "givenName": "Staš "
+ },
+ {
+ "id": "_:author_8",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Max Planck Institute for Polymer Research, Ackermannweg 10, 55128 Mainz, Germany"
+ },
+ "familyName": "Stuehn",
+ "givenName": "Torsten"
+ },
+ {
+ "id": "https://orcid.org/0000-0003-1480-6745",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Fraunhofer Institute for Algorithms and Scientific Computing, Schloss Birlinghoven, 53754 Sankt Augustin, Germany"
+ },
+ "familyName": "Reith",
+ "givenName": "Dirk "
+ }
+ ],
+ "codeRepository": "https://github.com/espressopp/espressopp.git",
+ "dateModified": "2024-01-25",
+ "description": "codecov\n\nESPResSo++ is an extensible, flexible, fast and parallel simulation software for soft matter research. It is a highly versatile software package for the scientific simulation and analysis of coarse-grained atomistic or bead-spring models as they are used in soft matter research. ESPResSo and ESPResSo++ have common roots and share parts of the developer/user community. However their development is independent and they are different software packages. ",
+ "downloadUrl": "https://github.com/espressopp/espressopp/archive/refs/tags/v3.1.0.tar.gz",
+ "keywords": [
+ "python",
+ "c-plus-plus",
+ "analysis",
+ "devtools",
+ "espresso",
+ "method",
+ "algorithm-challenges",
+ "adress",
+ "hpc-applications",
+ "lattice-boltzmann",
+ "algorithmic"
+ ],
+ "license": "https://spdx.org/licenses/GPL-3.0",
+ "name": " espressopp ",
+ "programmingLanguage": [
+ "C++",
+ "Python"
+ ],
+ "relatedLink": "http://www.espresso-pp.de/",
+ "version": "v3.1.0",
+ "issueTracker": "https://github.com/espressopp/espressopp/issues",
+ "referencePublication": "https://doi.org/10.1016/j.cpc.2012.12.004"
+}
diff --git a/data/normalization/SOFTNAME/gromacs/codemeta.json b/data/normalization/SOFTNAME/gromacs/codemeta.json
new file mode 100644
index 0000000..07fb457
--- /dev/null
+++ b/data/normalization/SOFTNAME/gromacs/codemeta.json
@@ -0,0 +1,57 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "_:author_1",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Bioson Research Institute and Laboratory of Biophysical Chemistry, The University of Groningen, Nijenborgh 4, 9747 AG Groningen, The Netherlands"
+ },
+ "familyName": "Berendsen",
+ "givenName": "Herman J.C. "
+ },
+ {
+ "id": "_:author_2",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Bioson Research Institute and Laboratory of Biophysical Chemistry, The University of Groningen, Nijenborgh 4, 9747 AG Groningen, The Netherlands"
+ },
+ "familyName": "van der Spoel",
+ "givenName": "David "
+ },
+ {
+ "id": "_:author_3",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Bioson Research Institute and Laboratory of Biophysical Chemistry, The University of Groningen, Nijenborgh 4, 9747 AG Groningen, The Netherlands"
+ },
+ "familyName": "van Drunen",
+ "givenName": "Rudi"
+ }
+ ],
+ "codeRepository": "https://github.com/gromacs/gromacs.git",
+ "dateModified": "2026-03-26",
+ "datePublished": "1995-09-02",
+ "description": "GROMACS is a versatile package to perform molecular dynamics, i.e. simulate the Newtonian equations of motion for systems with hundreds to millions of particles and is a community-driven project.",
+ "downloadUrl": "https://ftp.gromacs.org/gromacs/gromacs-2026.1.tar.gz",
+ "license": "https://spdx.org/licenses/LGPL-2.1",
+ "name": "gromacs",
+ "operatingSystem": "Unix",
+ "programmingLanguage": [
+ "C++",
+ "Cmake",
+ "Python",
+ "Cuda",
+ "C",
+ "Shell"
+ ],
+ "relatedLink": "https://www.gromacs.org/",
+ "version": "2026.1",
+ "issueTracker": "https://gitlab.com/gromacs/gromacs/-/issues",
+ "referencePublication": "DOI:10.1016/0010-4655(95)00042-E"
+}
diff --git a/data/normalization/SOFTNAME/lammps/codemeta.json b/data/normalization/SOFTNAME/lammps/codemeta.json
new file mode 100644
index 0000000..e90f538
--- /dev/null
+++ b/data/normalization/SOFTNAME/lammps/codemeta.json
@@ -0,0 +1,45 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "https://orcid.org/0000-0003-2446-8209",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Parallel Computational Sciences Department 1421, MS 1111, Sandia National Laboratories, Albuquerque, New Mexico 87185-1111"
+ },
+ "familyName": "Plimpton",
+ "givenName": "Steven J. "
+ }
+ ],
+ "codeRepository": "https://github.com/lammps/lammps.git",
+ "dateModified": "2026-03-30",
+ "datePublished": "2002-05-25",
+ "description": "LAMMPS is a classical molecular dynamics simulation code designed to\nrun efficiently on parallel computers. ",
+ "downloadUrl": "https://download.lammps.org/tars/lammps.tar.gz",
+ "funder": {
+ "type": "Organization",
+ "name": " US Department of Energy"
+ },
+ "keywords": [
+ "simulation",
+ "molecular-dynamics",
+ "lammps",
+ "kokkos"
+ ],
+ "license": "https://spdx.org/licenses/GPL-2.0",
+ "name": "lammps",
+ "programmingLanguage": [
+ "C++",
+ "Tcl",
+ "C",
+ "Cuda",
+ "Python",
+ "CMake"
+ ],
+ "relatedLink": "http://www.lammps.org/",
+ "issueTracker": "https://github.com/lammps/lammps/issues",
+ "referencePublication": "https://doi.org/10.1557/JMR.1995.1589"
+}
diff --git a/data/normalization/SOFTNAME/lassohtp/codemeta.json b/data/normalization/SOFTNAME/lassohtp/codemeta.json
new file mode 100644
index 0000000..7b20494
--- /dev/null
+++ b/data/normalization/SOFTNAME/lassohtp/codemeta.json
@@ -0,0 +1,42 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "codeRepository": "https://github.com/ChemBioHTP/LassoHTP.git",
+ "contributor": [
+ {
+ "id": "_:contributor_1",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Vanderbilt University "
+ },
+ "familyName": "Juarez ",
+ "givenName": "Reecan J"
+ },
+ {
+ "id": "_:contributor_2",
+ "type": "Person",
+ "familyName": "Yang",
+ "givenName": "Zhongyue John"
+ },
+ {
+ "id": "_:contributor_3",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Vanderbilt University"
+ },
+ "familyName": "Shao",
+ "givenName": "QZ"
+ },
+ {
+ "id": "_:contributor_4",
+ "type": "Person",
+ "givenName": "KleinesMesser"
+ }
+ ],
+ "description": "LassoHTP is a computational tool geared towards lasso peptide design and discovery. Major functionalities include the high throughput (HTP) construction of lasso peptides as well as HTP molecular dynamics (MD) simulations to predict lasso peptide thermodynamic parameters.",
+ "name": " LassoHTP ",
+ "programmingLanguage": "Python"
+}
diff --git a/data/normalization/SOFTNAME/plumed/codemeta.json b/data/normalization/SOFTNAME/plumed/codemeta.json
new file mode 100644
index 0000000..f6a4a9d
--- /dev/null
+++ b/data/normalization/SOFTNAME/plumed/codemeta.json
@@ -0,0 +1,144 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "https://orcid.org/0000-0002-7321-0004",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Computational Science, Department of Chemistry and Applied Biosciences, ETH Zurich, USI Campus, via G. Buffi 13, 6900 Lugano, Switzerland"
+ },
+ "familyName": "Bonomi",
+ "givenName": " Massimiliano "
+ },
+ {
+ "id": "_:author_2",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Computational Science, Department of Chemistry and Applied Biosciences, ETH Zurich, USI Campus, via G. Buffi 13, 6900 Lugano, Switzerland"
+ },
+ "familyName": "Branduardi",
+ "givenName": " Davide "
+ },
+ {
+ "id": "https://orcid.org/0000-0001-9216-5782",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Università di Modena e Reggio Emilia and INFM-CNR-S3, via Campi 213/A, 41100 Modena, Italy"
+ },
+ "familyName": "Bussi",
+ "givenName": "Giovanni "
+ },
+ {
+ "id": "https://orcid.org/0000-0002-9923-8590",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Physics, University of Milan and INFN sez. Milano, via Celoria 16, Milano 20133, Italy"
+ },
+ "familyName": "Camilloni",
+ "givenName": "Carlo "
+ },
+ {
+ "id": "_:author_5",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Dept. of Structural and Chemical Biology, Mount Sinai School of Medicine, 1425 Madison Avenue, New York, NY 10029-6574, USA"
+ },
+ "familyName": "Provasi",
+ "givenName": "Davide "
+ },
+ {
+ "id": "https://orcid.org/0000-0003-0692-0505",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "NRI and Dept. of Applied Chemistry, GPO Box U1987, 6845 Perth (WA), Australia"
+ },
+ "familyName": "Raiteri",
+ "givenName": " Paolo "
+ },
+ {
+ "id": "https://orcid.org/0000-0002-2150-4182",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Chemistry, University of California Davis, One Shields Avenue, Davis, CA 95616, USA"
+ },
+ "familyName": "Donadio",
+ "givenName": " Davide "
+ },
+ {
+ "id": "https://orcid.org/0000-0003-0044-6718",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "International School for Advanced Studies (SISSA), Via Beirut 2-4, 34014 Trieste, Italy"
+ },
+ "familyName": "Marinelli",
+ "givenName": "Fabrizio "
+ },
+ {
+ "id": "https://orcid.org/0000-0002-4892-2667",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "International School for Advanced Studies (SISSA), Via Beirut 2-4, 34014 Trieste, Italy"
+ },
+ "familyName": "Pietrucci",
+ "givenName": "Fabio "
+ },
+ {
+ "id": "_:author_10",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Department of Physics, University of Milan and INFN sez. Milano, via Celoria 16, Milano 20133, Italy / he Niels Bohr Institute, University of Copenhagen, Blegdamsvej 17, DK-2100 Copenhagen, Denmark"
+ },
+ "familyName": "Broglia ",
+ "givenName": " Ricardo A. "
+ },
+ {
+ "id": "_:author_11",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Computational Science, Department of Chemistry and Applied Biosciences, ETH Zurich, USI Campus, via G. Buffi 13, 6900 Lugano, Switzerland"
+ },
+ "familyName": "Parrinello",
+ "givenName": "Michele "
+ }
+ ],
+ "codeRepository": "https://github.com/plumed/plumed2.git",
+ "dateModified": "2025-08-25",
+ "description": "PLUMED is an open-source, community-developed library that provides a wide range of different methods, which include:\n\n enhanced-sampling algorithms\n free-energy methods\n tools to analyze the vast amounts of data produced by molecular dynamics (MD) simulations.\n\nThese techniques can be used in combination with a large toolbox of collective variables that describe complex processes in physics, chemistry, material science, and biology.\n\nPLUMED works together with some of the most popular MD engines, such as ACEMD, Amber, DL_POLY, GROMACS, LAMMPS, NAMD, OpenMM, DFTB+, ABIN, CP2K, i-PI, PINY-MD, and Quantum Espresso. In addition, PLUMED can be used to augment the capabilities of analysis tools such as VMD, HTMD, OpenPathSampling, and as a standalone utility to analyze pre-calculated MD trajectories.",
+ "downloadUrl": "https://github.com/plumed/plumed2/releases/download/v2.10.0/plumed-2.10.0.tgz",
+ "keywords": [
+ "plugin",
+ "c-plus-plus",
+ "molecular-dynamics",
+ "free-energy",
+ "trajectory-analysis",
+ "enhanced-sampling",
+ "plumed",
+ "plumed2"
+ ],
+ "license": "https://spdx.org/licenses/LGPL-3.0",
+ "name": " PLUMED ",
+ "programmingLanguage": [
+ "C++",
+ "Fortran",
+ "Python",
+ "HTML",
+ "Shell",
+ "Makefile"
+ ],
+ "relatedLink": "https://www.plumed.org//",
+ "version": "v2.10.0",
+ "referencePublication": "https://doi.org/10.48550/arXiv.0902.0874"
+}
diff --git a/data/normalization/SOFTNAME/probis/codemeta.json b/data/normalization/SOFTNAME/probis/codemeta.json
new file mode 100644
index 0000000..1f95f7b
--- /dev/null
+++ b/data/normalization/SOFTNAME/probis/codemeta.json
@@ -0,0 +1,38 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "https://orcid.org/0000-0003-0160-3375",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "National Institute of Chemistry, Hajdrihova 19, 1000 Ljubljana, Slovenia"
+ },
+ "familyName": "Konc",
+ "givenName": "Janez "
+ },
+ {
+ "id": "https://orcid.org/0000-0003-4067-0116",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "National Institute of Chemistry, Hajdrihova 19, 1000 Ljubljana, Slovenia"
+ },
+ "familyName": "Janezic",
+ "givenName": "Dusanka "
+ }
+ ],
+ "codeRepository": "https://gitlab.com/janezkonc/probis.git",
+ "description": "ProBiS algorithm aligns and superimposes complete protein surfaces, surface motifs, or protein binding sites. It enables pairwise alignments of entire protein structures or selected binding sites as well as fast database searches for similar protein binding sites. The algorithm can find similar binding sites even in proteins with different folds and without prior knowledge of their location. ProBiS algorithm is parallel for single or multiple CPU platforms. ",
+ "downloadUrl": "https://gitlab.com/janezkonc/probis/-/archive/master/probis-master.tar.gz?ref_type=heads",
+ "license": "https://spdx.org/licenses/BSD-3-Clause",
+ "name": "probis",
+ "operatingSystem": [
+ "Linux",
+ "Ubuntu"
+ ],
+ "relatedLink": "http://probis.cmm.ki.si/",
+ "referencePublication": "https://doi.org/10.1093/nar/gkq479"
+}
diff --git a/data/normalization/SOFTNAME/vmd/codemeta.json b/data/normalization/SOFTNAME/vmd/codemeta.json
new file mode 100644
index 0000000..16be0e9
--- /dev/null
+++ b/data/normalization/SOFTNAME/vmd/codemeta.json
@@ -0,0 +1,42 @@
+{
+ "@context": "https://w3id.org/codemeta/3.0",
+ "type": "SoftwareSourceCode",
+ "applicationCategory": "molecular dynamics",
+ "author": [
+ {
+ "id": "_:author_1",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Theoretical Biophysics Group, University of Illinois and Beckman Institute, Urbana, Illinois 61801, USA"
+ },
+ "familyName": "Humphrey",
+ "givenName": "William "
+ },
+ {
+ "id": "_:author_2",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Theoretical Biophysics Group, University of Illinois and Beckman Institute, Urbana, Illinois 61801, USA"
+ },
+ "familyName": "Dalke",
+ "givenName": " Andrew "
+ },
+ {
+ "id": "_:author_3",
+ "type": "Person",
+ "affiliation": {
+ "type": "Organization",
+ "name": "Theoretical Biophysics Group, University of Illinois and Beckman Institute, Urbana, Illinois 61801, USA"
+ },
+ "familyName": "Schulten",
+ "givenName": " Klaus "
+ }
+ ],
+ "dateModified": "2026-03-25",
+ "description": "VMD is a molecular visualization program for displaying, animating, and analyzing large biomolecular systems using 3-D graphics and built-in scripting. VMD supports computers running MacOS X, Unix, or Windows, is distributed free of charge, and includes source code. ",
+ "name": "Visual Molecular Dynamics",
+ "version": "2.0.0",
+ "referencePublication": "https://doi.org/10.1016/0263-7855(96)00018-5"
+}
diff --git a/data/normalization/md_forcefields_registry.json b/data/normalization/md_forcefields_registry.json
new file mode 100644
index 0000000..5f0cc05
--- /dev/null
+++ b/data/normalization/md_forcefields_registry.json
@@ -0,0 +1,566 @@
+{
+ "force_fields":
+ [
+ {"name": "Charmm 36",
+ "aliases": ["charmm36", "charmmc36", "c36"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000132",
+ "family": "charmm",
+ "category": "force field"
+ },
+ {"name": "Charmm 36m",
+ "aliases": ["charmm36m","c36m", "charmm c36"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field",
+ "publication": "https://doi.org/10.1038/nmeth.4067"
+ },
+ {"name": "CHARMM36-WYF",
+ "aliases": ["charm36-wyf"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field",
+ "publication": "https://doi.org/10.1021/acs.jctc.8b00839"
+ },
+ {"name": "Charmm TIP3P",
+ "aliases": ["charmmtip3p"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field",
+ "publication": null
+ },
+ {"name": "Charmm 27",
+ "aliases": ["charmm27"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000753",
+ "family": "charmm",
+ "category": "force field"
+ },
+ {"name": "Charmm 22*",
+ "aliases": ["charmm22*", "c22*"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field"
+ },
+ {"name": "Charmm 22",
+ "aliases": ["charmm22"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field",
+ "publication": "https://doi.org/10.1021/jp973084f"
+ },
+ {"name": "Charmm 36 CUFIX",
+ "aliases": ["charmm36cufix", "c36mcu"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field",
+ "publication": "https://doi.org/10.1021/jp973084f"
+ },
+ {"name": "Charmm36 (v. june 2015)",
+ "aliases": ["charmm36(v.june 2015)"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "charmm",
+ "category": "force field",
+ "publication": "https://doi.org/10.1021/jp973084f"
+ },
+ {"name": "Charmm",
+ "aliases": ["charmm"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000248",
+ "family": "charmm",
+ "category": "force field"
+ },
+ {"name": "TIP3P",
+ "aliases": ["tip3p", "charmmtip3p"],
+ "resolution": "all-atom",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000022",
+ "family": "water",
+ "category": "model"
+ },
+ {"name": "SPC/E",
+ "aliases": ["spc/e"],
+ "resolution": "all-atom",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000771",
+ "family": "water",
+ "category": "model"
+ },
+ {"name": "SPC",
+ "aliases": ["spc"],
+ "resolution": "all-atom",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000769",
+ "family": "water",
+ "category": "model"
+ },
+ {"name": "OPC",
+ "aliases": ["opc"],
+ "resolution": "all-atom",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000031",
+ "family": "water",
+ "category": "model"
+ },
+ {"name": "TIP4P/2005",
+ "aliases": ["tip4p/2005"],
+ "resolution": "all-atom",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000766",
+ "family": "water",
+ "category": "model"
+ },
+ {"name": "WatFour",
+ "aliases": ["watfour", "WT4"],
+ "resolution": "coarse-grain",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001493",
+ "family": "water",
+ "category": "model"
+ },
+ {"name": "Gromos 53a6",
+ "aliases": ["gromos53a6", "53a6"],
+ "resolution": "all-atom",
+ "molecular_type": "water",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001974",
+ "family": "gromos",
+ "category": "force field"
+ },
+ {"name": "Gromos 54a7",
+ "aliases": ["gromos54a7", "54a7"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "gromos",
+ "category": "force field"
+ },
+ {"name": "Gromos 87",
+ "aliases": ["gromos87"],
+ "resolution": "united-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001977",
+ "family": "gromos",
+ "category": "force field"
+ },
+ {"name": "Gromos 43a1-s3",
+ "aliases": ["gromos43a1-s3","43a1-s3"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "gromos",
+ "category": "force field"
+ },
+ {"name": "Gromos 56a6carbo/carbo_r",
+ "aliases": ["gromos56a6carbo/carbo_r"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "gromos",
+ "category": "force field"
+ },
+ {"name": "AMBER ff03 w",
+ "aliases": ["amberff03w", "ff03w", "amber03w"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff03*",
+ "aliases": ["amberff03*", "ff03*"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff03",
+ "aliases": ["amberff03", "ff03", "amber03"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001442",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER",
+ "aliases": ["amber"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "Amber ff14SB",
+ "aliases": ["amberff14SB", "amber14sb"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000021",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "Amber ff15ipq",
+ "aliases": ["amberff15ipq"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff99SB-ILDN",
+ "aliases": ["amberff99sb-ildn", "amberff99sb-ildn", "ff99sb-ildn", "amber99sb-ildb"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000750",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff99SB-ILDN-phi",
+ "aliases": ["amberf99sb-ildn-phi","ff99sb-ildn-phi" ],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff99SB-ILDN-nmr",
+ "aliases": ["amberff99sb-ildn-nmr","ff99sb-ildn-nmr"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff99SB-ILDN",
+ "aliases": ["amber99sb-ildn","amber99sb-ildb"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff99sb*",
+ "aliases": ["amber99sb*","ff99sb*"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff99",
+ "aliases": ["amberff99","ff99"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "AMBER ff96",
+ "aliases": ["amber96","ff96"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000745",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "amber10:extended huckel theory (eht)",
+ "aliases": ["amber10extendedhuckeltheory(eht)","amber10"],
+ "resolution": "all-atom",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000745",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "Lipid 21",
+ "aliases": ["lipid21", "amber lipid21"],
+ "resolution": "all-atom",
+ "molecular_type": "lipid",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000018",
+ "family": "amber",
+ "publication":"https://doi.org/10.1021/acs.jctc.1c01217",
+ "category": "force field"
+ },
+ {"name": "Lipid 17",
+ "aliases": ["lipid17", "amber lipid17"],
+ "resolution": "all-atom",
+ "molecular_type": "lipid",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000020",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "Lipid 14",
+ "aliases": ["lipid14", "amber lipid14"],
+ "resolution": "all-atom",
+ "molecular_type": "lipid",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000019",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "Slipids",
+ "aliases": ["slipids","sl"],
+ "resolution": null,
+ "molecular_type": "lipid",
+ "ontology_link": null,
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "Berger",
+ "aliases": ["berger"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "Trappe",
+ "aliases": ["trappe"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "RSFF2",
+ "aliases": ["rsff2"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "publication": "https://doi.org/10.1021/jp5064676",
+ "category": "force field"
+ },
+ {"name": "RSFF1",
+ "aliases": ["rsff1"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "publication": null,
+ "category": "force field"
+ },
+ {"name": "OPLS",
+ "aliases": ["opls"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000758",
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "OPLS-AA/L",
+ "aliases": ["l-opls"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001984",
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "OPLS-AA",
+ "aliases": ["opls-aa"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000759",
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "Martini 3",
+ "aliases": ["martini3", "martinini3.0", "martini 3.0"],
+ "resolution": "coarse-grain",
+ "molecular_type": null,
+ "ontology_link": null,
+ "publication": "https://doi.org/10.1038/s41592-021-01098-3",
+ "family": "martini",
+ "category": "force field"
+ },
+ {"name": "Martini 2",
+ "aliases": ["martini2", "martinini2.0", "martini 2.0"],
+ "resolution": "coarse-grain",
+ "molecular_type": null,
+ "ontology_link": null,
+ "publication": "https://doi.org/10.1021/ct700324x",
+ "family": "martini",
+ "category": "force field"
+ },
+ {"name": "Martini",
+ "aliases": ["martini"],
+ "resolution": "coarse-grain",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001982",
+ "publication": "https://doi.org/10.1021/jp071097f",
+ "family": "martini",
+ "category": "force field"
+ },
+ {"name": "SIRAH",
+ "aliases": ["sirah"],
+ "resolution": "coarse-grain",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000645",
+ "family": "sirah",
+ "category": "force field"
+ },
+ {"name": "GAFF",
+ "aliases": ["generalamberforcefield", "generalizedamberforcefield", "generalized amber force field", "general amber force field"],
+ "resolution": "coarse-grain",
+ "molecular_type": null,
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_000023",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "GAFFLipid",
+ "aliases": ["gafflipid"],
+ "resolution": "all-atom",
+ "molecular_type": "lipid",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001450",
+ "family": "amber",
+ "category": "force field"
+ },
+ {"name": "ECC-popc",
+ "aliases": ["ecc-popc"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "category": "variation"
+ },
+ {"name": "ECC-lipids",
+ "aliases": ["ecc-lipids"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "category": "variation"
+ },
+ {"name": "ECC-ions",
+ "aliases": ["ecc-ions"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "category": "variation"
+ },
+ {"name": "compass",
+ "aliases": ["compass"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "βol15",
+ "aliases": ["βol15"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "amber OL15",
+ "aliases": ["amberol15", "ol15"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null,
+ "category": "force field"
+ },
+ {"name": "trappe-ua",
+ "aliases": ["trappe-ua"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "Williams 7B",
+ "aliases": ["williams7b"],
+ "resolution": null,
+ "molecular_type": "variation",
+ "ontology_link": null,
+ "family": null,
+ "category": "variation"
+ },
+ {"name": "Williams",
+ "aliases": ["williams"],
+ "resolution": null,
+ "molecular_type": "variation",
+ "ontology_link": null,
+ "family": null,
+ "category": "variation"
+ },
+ {"name": "Ulmschneider",
+ "aliases": ["ulmschneider"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "PYS",
+ "aliases": ["pys"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "poger gromos 53a6_l",
+ "aliases": ["pogergromos53a6_l"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "PLAFF3",
+ "aliases": ["plaff3"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": "PARMBSC0",
+ "aliases": ["parmbsc0"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber"
+ },
+ {"name": "PARM99LNA",
+ "aliases": ["parm99lna", "parm99_lna"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": "amber"
+ },
+ {"name": "NB-FIX",
+ "aliases": ["nbfix"],
+ "resolution": null,
+ "molecular_type": "variation",
+ "ontology_link": "http://purl.obolibrary.org/obo/MOLSIM_001545",
+ "family": null,
+ "category": "variation"
+ },
+ {"name": "DANG",
+ "aliases": ["dang"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ },
+ {"name": " BPY1,3-TFSI",
+ "aliases": ["bpy1,3-tfsi"],
+ "resolution": null,
+ "molecular_type": null,
+ "ontology_link": null,
+ "family": null
+ }
+ ]
+}
diff --git a/src/mdner_llm/models/entities.py b/src/mdner_llm/models/entities.py
index 26bfb9f..37f943c 100644
--- a/src/mdner_llm/models/entities.py
+++ b/src/mdner_llm/models/entities.py
@@ -98,87 +98,3 @@ class ListOfEntities(BaseModel):
| SoftwareName
| SoftwareVersion
] = Field(..., description="List of recognized entities extracted from text.")
-
-
-# =====================================================================
-# Normalized Entity Models
-# =====================================================================
-class NormalizedEntity(Entity):
- """Represents an entity with additional normalization and validation fields."""
-
- text_normalized: str = Field(
- ..., description="Normalized version of the extracted text."
- )
- is_hallucinated: bool = Field(
- ..., description="True if the entity text is absent from the source text."
- )
-
-
-# =====================================================================
-# Entity subclasses for each specific annotation type
-# =====================================================================
-class MoleculeNormalized(NormalizedEntity):
- """Entity representing a normalized molecule, protein, lipid, or similar object."""
-
- category: Literal["MOL"] = Field(
- "MOL", description="Category for molecule entities."
- )
-
-
-class SimulationTimeNormalized(NormalizedEntity):
- """Entity representing a normalized simulation time duration (e.g., 50 ns, 5 ms)."""
-
- category: Literal["STIME"] = Field(
- "STIME", description="Category for simulation time entities."
- )
-
-
-class ForceFieldModelNormalized(NormalizedEntity):
- """Entity representing a normalized force field used in the MD simulation."""
-
- category: Literal["FFM"] = Field(
- "FFM", description="Category for force field or model entities."
- )
-
-
-class SimulationTemperatureNormalized(NormalizedEntity):
- """Entity representing a normalized temperature value used in the simulation."""
-
- category: Literal["STEMP"] = Field(
- "STEMP", description="Category for simulation temperature entities."
- )
-
-
-class SoftwareNameNormalized(NormalizedEntity):
- """Entity representing the normalized name of software used for simulations."""
-
- category: Literal["SOFTNAME"] = Field(
- "SOFTNAME", description="Category for software name entities."
- )
-
-
-class SoftwareVersionNormalized(NormalizedEntity):
- """Entity representing the normalized version of a software package."""
-
- category: Literal["SOFTVERS"] = Field(
- "SOFTVERS", description="Category for software version entities."
- )
-
-
-# =====================================================================
-# Container class for all extracted entities
-# =====================================================================
-class ListOfEntitiesNormalized(BaseModel):
- """Structured list of all extracted and normalized entities."""
-
- entities: list[
- MoleculeNormalized
- | SimulationTimeNormalized
- | ForceFieldModelNormalized
- | SimulationTemperatureNormalized
- | SoftwareNameNormalized
- | SoftwareVersionNormalized,
- ] = Field(
- ...,
- description="List of recognized and normalized entities extracted from text.",
- )
diff --git a/src/mdner_llm/models/entities_normalized.py b/src/mdner_llm/models/entities_normalized.py
new file mode 100644
index 0000000..773375d
--- /dev/null
+++ b/src/mdner_llm/models/entities_normalized.py
@@ -0,0 +1,197 @@
+"""Pydantic model for entities extracted by an LLM for NER tasks."""
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from mdner_llm.models.entities import Entity
+
+
+class Affiliation(BaseModel):
+ """Organization affiliation details."""
+
+ type: str = Field(default="Organization", description="Type of organization.")
+ name: str = Field(description="Full name and address of the organization.")
+
+
+class Person(BaseModel):
+ """Author details from CodeMeta schema."""
+
+ id: str = Field(description="ORCID URL or internal unique identifier.")
+ type: str = Field(default="Person", description="Entity type, always 'Person'.")
+ first_name: str = Field(description="First/given name of the author.")
+ last_name: str = Field(description="Last/family name of the author.")
+ affiliation: Affiliation | None = Field(
+ default=None, description="Organization affiliation."
+ )
+
+
+# =====================================================================
+# Normalized Entity Models
+# =====================================================================
+class NormalizedEntity(Entity):
+ """Represents an entity with additional normalization and validation fields."""
+
+ text_normalized: str = Field(
+ ..., description="Normalized version of the extracted text."
+ )
+ is_hallucinated: bool = Field(
+ ..., description="True if the entity text is absent from the source text."
+ )
+
+
+# =====================================================================
+# Entity subclasses for each specific annotation type
+# =====================================================================
+class MoleculeNormalized(NormalizedEntity):
+ """Entity representing a normalized molecule, protein, lipid, or similar object."""
+
+ category: Literal["MOL"] = Field(
+ "MOL", description="Category for molecule entities."
+ )
+
+
+class SimulationTimeNormalized(NormalizedEntity):
+ """Entity representing a normalized simulation time duration (e.g., 50 ns, 5 ms)."""
+
+ category: Literal["STIME"] = Field(
+ "STIME", description="Category for simulation time entities."
+ )
+ value: float | None = Field(
+ default=None,
+ description="Numeric value of the simulation time.",
+ )
+ unit: str | None = Field(
+ default=None,
+ description="The time unit in a standardized format (e.g., 'ns', 'μs', 's').",
+ )
+
+
+class ForceFieldModelNormalized(NormalizedEntity):
+ """Entity representing a normalized force field used in the MD simulation."""
+
+ category: Literal["FFM"] = Field(
+ "FFM", description="Category for force field or model entities."
+ )
+ tag: Literal["force field", "model", "variation"] | None = Field(
+ default=None,
+ description="Classification flag: either 'force field' or 'model'.",
+ )
+ family: str | None = Field(
+ default=None,
+ description="Family or category of the force field (e.g., 'AMBER', 'CHARMM').",
+ )
+ aliases: list[str] | None = Field(
+ default=None,
+ description="List of alternative names or aliases for the force field.",
+ )
+ resolution: Literal["all-atom", "coarse-grain", "united-atom"] | None = Field(
+ default=None,
+ description="The structural resolution of the force field model.",
+ )
+ molecular_type: str | None = Field(
+ default=None,
+ description="The type of molecules the force field is designed "
+ "for (e.g., 'proteins', 'lipids').",
+ )
+ ontology_link: str | None = Field(
+ default=None,
+ description="URL mapping the force field entity to the MOLSIM ontology. "
+ "Documentation: https://bioportal.bioontology.org/ontologies/MOLSIM",
+ )
+ publication_link: str | None = Field(
+ default=None,
+ description="URL linking to the primary publication or reference.",
+ )
+
+
+class SimulationTemperatureNormalized(NormalizedEntity):
+ """Entity representing a normalized temperature value used in the simulation."""
+
+ category: Literal["STEMP"] = Field(
+ "STEMP", description="Category for simulation temperature entities."
+ )
+ value: float | None = Field(
+ default=None,
+ description="Numeric value of the temperature if available, otherwise None.",
+ )
+ unit: str | None = Field(
+ default=None,
+ description="The temperature unit after normalization. "
+ "All input units (e.g., '°C') are converted to Kelvin ('K').",
+ )
+
+
+class SoftwareNameNormalized(NormalizedEntity):
+ """Entity representing the normalized name of software used for simulations."""
+
+ category: Literal["SOFTNAME"] = Field(
+ "SOFTNAME", description="Category for software name entities."
+ )
+ name: str = Field(description="The canonical name of the software (e.g., 'amber').")
+ authors: list[Person] | None = Field(
+ default=None,
+ description="List of authors/contributors associated with the software.",
+ )
+ description: str | None = Field(
+ default=None, description="Brief summary or description of the software suite."
+ )
+ version: str | None = Field(
+ default=None, description="The specific software release version."
+ )
+ date_last_modification: str | None = Field(
+ default=None, description="The last modification or update date (YYYY-MM-DD)."
+ )
+ code_repository_link: str | None = Field(
+ default=None,
+ description="URL to the official source code repository (e.g., GitHub).",
+ )
+ download_url: str | None = Field(
+ default=None,
+ description="Direct link to download the software source package archive.",
+ )
+ related_link: str | None = Field(
+ default=None, description="URL to the primary project website or home page."
+ )
+ publication_link: str | None = Field(
+ default=None,
+ description="DOI or URL linking to the foundational reference paper.",
+ )
+ license: str | None = Field(
+ default=None,
+ description="SPDX license identifier or URL governing the software use.",
+ )
+ keywords: str | list[str] | None = Field(
+ default=None, description="Keywords or tags characterizing the software domain."
+ )
+ programming_language: list[str] | None = Field(
+ default=None,
+ description="List of primary programming languages used in the repository.",
+ )
+
+
+class SoftwareVersionNormalized(NormalizedEntity):
+ """Entity representing the normalized version of a software package."""
+
+ category: Literal["SOFTVERS"] = Field(
+ "SOFTVERS", description="Category for software version entities."
+ )
+
+
+# =====================================================================
+# Container class for all extracted entities
+# =====================================================================
+class ListOfEntitiesNormalized(BaseModel):
+ """Structured list of all extracted and normalized entities."""
+
+ entities: list[
+ MoleculeNormalized
+ | SimulationTimeNormalized
+ | ForceFieldModelNormalized
+ | SimulationTemperatureNormalized
+ | SoftwareNameNormalized
+ | SoftwareVersionNormalized,
+ ] = Field(
+ ...,
+ description="List of recognized and normalized entities extracted from text.",
+ )
diff --git a/src/mdner_llm/normalization/evaluate_llm_models.py b/src/mdner_llm/normalization/evaluate_llm_models.py
new file mode 100644
index 0000000..1b2f42e
--- /dev/null
+++ b/src/mdner_llm/normalization/evaluate_llm_models.py
@@ -0,0 +1,464 @@
+"""Script to normalize the simlation times into standard units."""
+
+import json
+import os
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+import click
+import instructor
+from dotenv import load_dotenv
+from instructor.core.exceptions import ValidationError
+from instructor.exceptions import InstructorRetryException
+from loguru import logger
+from openai import OpenAI
+from pydantic import BaseModel, Field
+
+load_dotenv()
+
+# list of model that we are going to test
+MODELS = [
+ "openai/gpt-4o",
+ "openai/gpt-5.5",
+ "deepseek/deepseek-v4-pro",
+ "google/gemma-4-31b-it",
+ "qwen/qwen3.6-27b",
+ "minimax/minimax-m2.7",
+ "moonshotai/kimi-k2.6",
+ "anthropic/claude-opus-4.7",
+ "mistralai/mistral-large-2512",
+]
+
+
+class SimulationTime(BaseModel):
+ """Define the structure of simulation time entity."""
+
+ value: float | None = Field(
+ ..., description="Normalized value ofthe simulation time"
+ )
+ unit: Literal["ps", "ns", "μs", "ms", "s"] | None = Field(
+ ..., max_length=2, description="Normalized unit of the simulation time"
+ )
+
+
+class NormSimuTime(BaseModel):
+ """Define the structure for the output of the LLM."""
+
+ input: str = Field(..., description="raw value of one simulaton time")
+ output: list[SimulationTime] = Field(
+ ..., description="normalized simulation timevalues and units"
+ )
+
+
+# We load the simulation times from the file in a list of simulatuion time to enable
+# slicing the list
+def load_simulation_times(ground_truth_file: Path) -> list:
+ """Load simulation times from a file into a list.
+
+ Parameters
+ ----------
+ ground_truth_file (Path): Path to the input file containing the simulation times
+
+ Returns
+ -------
+ list: A list of simulation times loaded from the file
+ """
+ logger.info(f"Loading the simulation times from {ground_truth_file}...")
+ times = []
+ with open(ground_truth_file) as gt_file:
+ ground_truth = json.load(gt_file)
+ for value in ground_truth["groundtruth"]:
+ times.append(value["input"])
+ logger.success(f"Loaded {len(times)} simulation times successfully.")
+ return times
+
+
+# We load the prompt text from the file
+def load_prompt_text(prompt_file_path):
+ """Extract the prompt from the txt prompt file.
+
+ Parameters
+ ----------
+ prompt_file_path(Path): Path to the file containing the prompt
+
+ Returns
+ -------
+ (str): content of the prompt file
+ """
+ content = ""
+ content = Path(prompt_file_path).read_text()
+ return content
+
+
+# We give to a chosen model a normalisation time, the model is called via an
+# openrouter key and use instructor to ensure the structured output of the llm.
+# We retrieve the time and the cost of the normalisation
+def normalize_simulation_time(
+ raw_simulation_time: str, model_name: str, prompt_file_path: Path
+):
+ """Normalize the units in the simulation time text to standard units.
+
+ Parameters
+ ----------
+ simulation_time_filepath: Path to the input text file containing simulation time
+ values.
+
+ Returns
+ -------
+ A string containing the normalized simulation time values in JSON format.
+ """
+ client = instructor.from_openai(
+ OpenAI(
+ base_url="https://openrouter.ai/api/v1",
+ api_key=os.getenv("OPEN_ROUTER_KEY"),
+ )
+ )
+
+ logger.info(f"{model_name.replace('-', '_')} | Normalizing: {raw_simulation_time}")
+ prompt = load_prompt_text(prompt_file_path)
+
+ try:
+ start_time = time.perf_counter()
+ completion_pydantic, completion_basic = (
+ client.chat.completions.create_with_completion(
+ model=model_name,
+ max_retries=3,
+ response_model=NormSimuTime,
+ messages=[
+ {
+ "role": "system",
+ "content": f"{prompt}",
+ },
+ {
+ "role": "user",
+ "content": f"{raw_simulation_time}",
+ },
+ ],
+ )
+ )
+ # total_cost = completion_basic.usage.cost_details["upstream_inference_cost"]
+ except InstructorRetryException as exc:
+ logger.error(f"Normalisation failed after {exc.n_attempts} attempts")
+ elapsed_time = time.perf_counter() - start_time
+ return None, elapsed_time, None
+
+ except ValidationError as exc:
+ logger.error(f"Pydantic validation failed: {exc}")
+ elapsed_time = time.perf_counter() - start_time
+ return None, elapsed_time, None
+
+ elapsed_time = time.perf_counter() - start_time
+ logger.info(
+ f"{model_name.replace('-', '_')} | output: {completion_pydantic.output}"
+ )
+ cost = completion_basic.usage.cost_details["upstream_inference_cost"]
+ return completion_pydantic.model_dump_json(), elapsed_time, cost
+
+
+# We format the output of the normalized simulation time in a json format.
+def format_norm_simulation_time(
+ raw_simulation_time: list, model_name: str, prompt_file_path: Path
+) -> dict:
+ """Format the normalized time to a JSON format with the normalized values.
+
+ Parameters
+ ----------
+ raw_simulation_time (Path) : Path to the input file containing
+ the raw simulation times
+ normalized_simulation_time (Path) : Path to the output file containing
+ the normalized simulation times
+
+ Returns
+ -------
+ dict[list] : dictonarry that contains the results of the simulation times
+ normalisation
+ """
+ all_simulation_times_norm = []
+ normalisation_output = {}
+ # We loop on the list of simulation times and normalize each entity
+ for simulation_time in raw_simulation_time:
+ normalization_result = normalize_simulation_time(
+ simulation_time, model_name, prompt_file_path
+ )
+ if normalization_result:
+ # We put the normalized simulation times in a json format
+ simulation_time_normalized = json.loads(normalization_result[0])
+ all_simulation_times_norm.append(simulation_time_normalized)
+ else:
+ simulation_time_not_normalized = {
+ "input": simulation_time,
+ "output": normalization_result,
+ }
+ all_simulation_times_norm.append(simulation_time_not_normalized)
+ normalisation_output["normalisation_output"] = all_simulation_times_norm
+ return normalisation_output
+
+
+def save_norm_simulation_results(
+ normalisation_output: dict, normalized_simulation_time: Path
+):
+ """Generate a JSON file with the results of the simulation times normalisation."""
+ logger.info("Saving the normalisation results in the JSON file")
+ with open(normalized_simulation_time, "w") as file_1:
+ json.dump(normalisation_output, file_1, indent=4, ensure_ascii=False)
+ # logger.success("Saving results to JSON file successful")
+
+
+def normalize_all_entities(
+ raw_simulation_times: list,
+ model: str,
+ ground_truth_dict: dict,
+ prompt_file_path: Path,
+) -> tuple[int, int, int]:
+ """
+ Normalize all the simulation times.
+
+ Parameters
+ ----------
+ raw_simulation_times (list): A list of raw simulation time strings to be normalized.
+ model (str): The name of the model to be used for normalization.
+ ground_truth_dict (dict): A dictionary mapping raw simulation time strings to their
+ corresponding ground truth normalized values.
+
+ Returns
+ -------
+ int: The number of correctly normalized simulation times compared to the ground
+ truth.
+ """
+ normalised_entity = 0
+ normalisation_time = 0
+ normalisation_cost = 0
+ entity_number = 1
+
+ for raw_simulation_time in raw_simulation_times:
+ logger.info("-" * 100)
+ logger.info(
+ f"{model.replace('-', '_')} | entity: {entity_number} / {len(raw_simulation_times)}"
+ )
+ normalisation_result = normalize_simulation_time(
+ raw_simulation_time,
+ model_name=model,
+ prompt_file_path=prompt_file_path,
+ )
+ if normalisation_result:
+ if normalisation_result[0]:
+ normalized_result_json = normalisation_result[0]
+ if normalisation_result[1]:
+ normalisation_time += normalisation_result[1]
+ if normalisation_result[2]:
+ normalisation_cost += normalisation_result[2]
+ if normalisation_result[0]:
+ normalized_data = json.loads(normalized_result_json)
+ ground_truth = ground_truth_dict.get(raw_simulation_time)
+
+ if ground_truth is None:
+ logger.warning(f"No ground truth found for: {raw_simulation_time}")
+ continue
+
+ match = True
+
+ if len(normalized_data["output"]) != len(ground_truth):
+ match = False
+ else:
+ for i in range(len(normalized_data["output"])):
+ if (
+ normalized_data["output"][i]["value"]
+ != ground_truth[i]["value"]
+ ):
+ logger.error(
+ f"{model.replace('-', '_')} | Normalisation failed "
+ )
+ match = False
+ break
+ if (
+ normalized_data["output"][i]["unit"]
+ != ground_truth[i]["unit"]
+ ):
+ logger.error(
+ f"{model.replace('-', '_')} | Normalisation failed "
+ )
+ match = False
+ break
+
+ if match:
+ logger.success(
+ f"{model.replace('-', '_')} | Normalisation successfull "
+ )
+ entity_number += 1
+ normalised_entity += 1
+ else:
+ entity_number += 1
+
+ # logger.info(f"entity: {normalised_entity} / {len(raw_simulation_times)}")
+ return normalised_entity, normalisation_time, normalisation_cost
+
+
+def evaluate_all_models(
+ raw_simulation_times: list,
+ ground_truth_file: Path,
+ runs: int,
+ prompt_file_path: Path,
+):
+ """Evaluate all models and save results to TSV file.
+
+ Parameters
+ ----------
+ raw_simulation_times (list): A list of raw simulation time strings to be normalized.
+ ground_truth_file (Path): Path to the ground truth JSON file containing the correct
+ normalized values for the simulation times.
+ runs (int): The number of runs to perform for each model to
+ calculate average accuracy.
+
+ Returns
+ -------
+ list[dict]: A list of dictionaries containing the model names and their
+ corresponding accuracy percentages.
+ """
+ with open(ground_truth_file) as f:
+ ground_truth_data = json.load(f)
+
+ ground_truth_dict = {}
+ for truth in ground_truth_data["groundtruth"]:
+ ground_truth_dict[truth["input"]] = truth["output"]
+
+ results = []
+
+ for model in MODELS:
+ logger.info("-" * 20)
+ logger.info(f"Model: {model.replace('-', '_')}")
+ total_correct = 0
+ total_normalisation_time = 0
+ total_normalisation_cost = 0
+
+ for run in range(runs):
+ logger.info("-" * 80)
+ logger.info(f"{model.replace('-', '_')} | Run {run + 1}/{runs}")
+
+ normalisation_results = normalize_all_entities(
+ raw_simulation_times, model, ground_truth_dict, prompt_file_path
+ )
+
+ normalised_entity = normalisation_results[0]
+ normalisation_time = normalisation_results[1]
+ normalisation_cost = normalisation_results[2]
+ run_accuracy = (normalised_entity / len(raw_simulation_times)) * 100
+
+ logger.info(f" Run accuracy: {run_accuracy:.1f}%")
+ total_correct += normalised_entity
+ total_normalisation_time += normalisation_time
+ total_normalisation_cost += normalisation_cost
+
+ accuracy = (total_correct / (len(raw_simulation_times) * runs)) * 100
+ normalisation_time_by_entity = total_normalisation_time / (
+ len(raw_simulation_times) * runs
+ )
+
+ results.append(
+ {
+ "model_name": model,
+ "accuracy_percentage": round(accuracy),
+ "inference_time_by_entity": round(normalisation_time_by_entity),
+ "inference_cost_by_entity_USD": round(
+ total_normalisation_cost / (len(raw_simulation_times) * runs)
+ ),
+ }
+ )
+
+ logger.info(
+ f"\n {model.replace('-', '_')} : Accuracy = {accuracy:.1f}% Time = {total_normalisation_time}"
+ f" Cost = {total_normalisation_cost / (len(raw_simulation_times) * runs)}\n"
+ )
+
+ return results
+
+
+def save_evaluation_results_in_tsv(
+ model_evaluation_file: Path,
+ raw_simulation_times: list,
+ ground_truth_file: Path,
+ prompt_file_path: Path,
+ runs: int,
+):
+ """Save the evaluation results of all models in a TSV file.
+
+ Parameters
+ ----------
+ model_evaluation_file (Path): Path to the TSV file for model evaluation results.
+ raw_simulation_times (list): A list of raw simulation time strings to be normalized.
+ ground_truth_file (Path): Path to the ground truth JSON file containing the correct
+ normalized values for the simulation times.
+ runs (int): The number of runs to perform for each model to calculate the
+ average accuracy.
+ """
+ results = evaluate_all_models(
+ raw_simulation_times, ground_truth_file, runs, prompt_file_path
+ )
+ with open(model_evaluation_file, "w") as f:
+ f.write(
+ "model_name\taccuracy_percentage\tnormalisation_times_sec\tnormalisation_cost\n"
+ )
+ f.writelines(
+ f"{result['model_name']}\t{result['accuracy_percentage']}\t{result['inference_time_by_entity']}\t{result['inference_cost_by_entity_USD']}\n"
+ for result in results
+ )
+
+
+@click.command()
+@click.option(
+ "--groundtruth-path",
+ type=click.Path(exists=True, file_okay=True, path_type=Path),
+ help="Path to the groundtruth file containing manually normalized simulation times",
+)
+@click.option(
+ "--runs",
+ default=10,
+ type=int,
+ help="Number of runs of the script",
+)
+@click.option(
+ "--model-evaluation-path",
+ type=click.Path(file_okay=True, path_type=Path),
+ help="Path to the TSV file for model evaluation results",
+)
+@click.option(
+ "--prompt-path",
+ type=click.Path(file_okay=True, path_type=Path),
+ help="Path to the llm prompt file",
+)
+def main_normalizing_simulation_times(
+ groundtruth_path: Path,
+ runs: int,
+ model_evaluation_path: Path,
+ prompt_path: Path,
+):
+ """Normalize the simulation times entities bu running all annexe functions."""
+ times = load_simulation_times(groundtruth_path)
+ times = times[:]
+ save_evaluation_results_in_tsv(
+ model_evaluation_path,
+ times,
+ groundtruth_path,
+ prompt_path,
+ runs,
+ )
+
+
+if __name__ == "__main__":
+ timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+ os.makedirs("logs", exist_ok=True)
+ logger_format = (
+ "{time:YYYY-MM-DD HH:mm:ss} "
+ "| {level:<8} "
+ "| {message}"
+ )
+ logger.remove()
+ logger.add(sys.stdout, format=logger_format, level="DEBUG")
+ logger.add(
+ f"logs/normalize_simulation_time{timestamp}.log",
+ level="DEBUG",
+ format=logger_format,
+ )
+ main_normalizing_simulation_times()
diff --git a/src/mdner_llm/normalization/ground_molecule_from_all_database.py b/src/mdner_llm/normalization/ground_molecule_from_all_database.py
new file mode 100644
index 0000000..e2ef0ee
--- /dev/null
+++ b/src/mdner_llm/normalization/ground_molecule_from_all_database.py
@@ -0,0 +1,1139 @@
+"""Script to normalize small molecule entities across various databases."""
+
+import re
+from pathlib import Path
+
+import httpx
+import pandas as pd
+from loguru import logger
+
+API_KEGG = "https://rest.kegg.jp"
+API_CHEBI = "https://www.ebi.ac.uk/chebi/backend/api/public/es_search/"
+API_PUBCHEM = "https://pubchem.ncbi.nlm.nih.gov/rest/pug"
+API_PDB = "https://data.rcsb.org/rest/v1/core/entry/"
+API_UNIPROT = "https://rest.uniprot.org/uniprotkb/"
+
+
+def get_type(entry: str) -> str:
+ """Determine the molecular entity type based on regex pattern.
+
+ Parameters
+ ----------
+ entry : str
+ The molecular identifier string to classify.
+
+ Returns
+ -------
+ str
+ The entity type: "PDB", "UNIPROT", "DNA", "RNA", "PROTEIN",
+ or "SMALL_MOLECULE" for anything else.
+ """
+ entry = entry.replace("`", "'")
+ entry = entry.replace("\u2019", "'")
+ logger.info(entry)
+
+ # PDB codes are 4 characters starting with a number
+ if re.search(r"^[1-9]([a-z]|[1-9]){3}$", entry) is not None:
+ return "PDB"
+
+ # UniProt accession pattern matching
+ if (
+ re.search(
+ r"[opq][0-9][a-z0-9]{3}[0-9]|[a-nr-z][0-9]([a-z][a-z0-9]{2}[0-9]){1,2}",
+ entry,
+ )
+ is not None
+ ):
+ return "UNIPROT"
+
+ # DNA sequence pattern (only a, t, c, g)
+ if re.search(r"^5'-[atcg]+-3'$", entry) is not None:
+ return "DNA"
+
+ # RNA sequence pattern
+ if re.search(r"^5'-[aucg]+-3'$", entry) is not None:
+ return "RNA"
+
+ # Amino acid sequence pattern
+ if (
+ re.search(r"^(?![ACGT]+$)[ACDEFGHIKLMNPQRSTVWY]{20,}$", entry) is not None
+ and len(entry) > 4
+ ):
+ return "PROTEIN"
+
+ return "SMALL_MOLECULE"
+
+
+def call_pdb(code_pdb: str) -> dict:
+ """Query the Protein Data Bank API for a given PDB code.
+
+ Parameters
+ ----------
+ code_pdb : str
+ 4-character PDB identifier code.
+
+ Returns
+ -------
+ dict
+ Details retrieved from the PDB database (entry_id, pubmed_id, doi, name).
+ """
+ logger.info(f"Searching for `{code_pdb}` in PDB database...")
+ try:
+ response = httpx.get(f"{API_PDB}{code_pdb}", timeout=200)
+ if response is not None and response.status_code == 200:
+ results = response.json()
+ logger.success(f"PDB grounding successful for `{code_pdb}`.")
+ return {
+ "entity_name": code_pdb,
+ "type": "PDB",
+ "id": results.get("rcsb_id", "Not Available"),
+ "name": results.get("struct", {}).get("title", "Not Available"),
+ }
+ status = response.status_code if response is not None else "network error"
+ logger.warning(
+ f"Failed to ground `{code_pdb}` in PDB database (HTTP {status})."
+ )
+ return {
+ "entity_name": code_pdb,
+ "type": "PDB",
+ "id": status,
+ "name": status,
+ }
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {f'{API_PDB}{code_pdb}'}: {e}")
+ return {
+ "entity_name": code_pdb,
+ "type": "PDB",
+ "error": "RemoteProtocolError",
+ }
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {f'{API_PDB}{code_pdb}'}: {e}")
+ return {
+ "entity_name": code_pdb,
+ "type": "PDB",
+ "error": "TimeoutException",
+ }
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {f'{API_PDB}{code_pdb}'}: {e}")
+ return {
+ "entity_name": code_pdb,
+ "type": "PDB",
+ "error": "RequestError",
+ }
+
+
+def call_uniprot(code_uniprot: str) -> dict:
+ """Query the UniProt API for a given UniProt accession code.
+
+ Parameters
+ ----------
+ code_uniprot : str
+ UniProt accession identifier.
+
+ Returns
+ -------
+ dict
+ Details retrieved from the UniProt database (accession, id, gene_name).
+ """
+ logger.info(f"Searching for `{code_uniprot}` in Uniprot database...")
+ try:
+ response = httpx.get(f"{API_UNIPROT}{code_uniprot}.json", timeout=200)
+ if response is not None and response.status_code == 200:
+ results = response.json()
+ logger.success(f"UniProt grounding successful for `{code_uniprot}`.")
+ return {
+ "entity_name": code_uniprot,
+ "type": "UNIPROT",
+ "id": results.get("primaryAccession"),
+ "name": results.get("genes", [{}])[0].get("geneName", {}).get("value"),
+ }
+ status = response.status_code if response is not None else "network error"
+ logger.warning(
+ f"Failed to ground `{code_uniprot}` in UniProt database (HTTP {status})."
+ )
+ return {
+ "entity_name": code_uniprot,
+ "type": "UNIPROT",
+ "id": status,
+ "name": status,
+ }
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {f'{API_UNIPROT}{code_uniprot}'}: {e}")
+ return {
+ "entity_name": code_uniprot,
+ "type": "UNIPROT",
+ "error": "RemoteProtocolError",
+ }
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {f'{API_UNIPROT}{code_uniprot}'}: {e}")
+ return {
+ "entity_name": code_uniprot,
+ "type": "UNIPROT",
+ "error": "TimeoutException",
+ }
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {f'{API_UNIPROT}{code_uniprot}'}: {e}")
+ return {
+ "entity_name": code_uniprot,
+ "type": "UNIPROT",
+ "error": "RequestError",
+ }
+
+
+def filter_molecules(molecules: list[str]) -> tuple[list[str], list[dict]]:
+ """Split molecules into small molecules and other entity types.
+
+ Small molecules are passed to the ChEBI/PubChem/KEGG pipeline.
+ PDB, UNIPROT, DNA, RNA, PROTEIN entities are queried and saved separately.
+
+ Parameters
+ ----------
+ molecules : list[str]
+ The full list of molecule entity names.
+
+ Returns
+ -------
+ tuple[list[str], list[dict]]
+ - list of small molecule names to pass to the chemical pipeline
+ - list of dicts for non-small-molecule entities (for TSV output)
+ """
+ # Only contains mol entities that will go throught a database consensus
+ small_molecules = []
+ # Contains PDB, Uniprot, and sequences
+ other_entities = []
+ logger.info(f"Filtering {len(molecules)} molecules ...")
+ for mol in molecules:
+ entity_type = get_type(mol)
+ if entity_type == "SMALL_MOLECULE":
+ small_molecules.append(mol)
+ elif entity_type == "PDB":
+ result = call_pdb(mol)
+ other_entities.append(result)
+ elif entity_type == "UNIPROT":
+ result = call_uniprot(mol)
+ other_entities.append(result)
+ else:
+ other_entities.append(
+ {
+ "entity_name": mol,
+ "type": entity_type,
+ "id": None,
+ "name": None,
+ }
+ )
+
+ logger.info(
+ f"Filtered {len(small_molecules)} small molecules and {len(other_entities)}"
+ f"other entities."
+ )
+ return small_molecules, other_entities
+
+
+def save_pdb_uniprot_seq_entities(
+ other_entities: list[dict],
+ output_file: Path,
+) -> None:
+ """Save PDB, UniProt, DNA, RNA, and PROTEIN entities in a TSV file.
+
+ Parameters
+ ----------
+ other_entities : list[dict]
+ List of dicts with keys: entity_name, type, id, name.
+ output_file : Path
+ The path to the output TSV file.
+ """
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+
+ with open(output_file, "w") as file:
+ file.write("Molecule\tType\tID\tName\n")
+ for entity in other_entities:
+ file.writelines(
+ f"{entity['entity_name']}\t{entity['type']}\t{entity['id']}\t{entity['name']}\n"
+ )
+
+ logger.info(
+ f"Saved {len(other_entities)} non-small-molecule entities to {output_file}."
+ )
+
+
+def query_kegg_by_name(
+ entity_name: str,
+) -> tuple[str, str] | tuple[None, None] | tuple[str, None] | tuple[None, str]:
+ """Return the KEGG ID linked to the molecule name.
+
+ Parameters
+ ----------
+ entity_name : str
+ The name of the molecule to query.
+
+ Returns
+ -------
+ tuple[str, str] | tuple[None, None] | tuple[str, None] | tuple[None, str]
+ A tuple containing the pubchem ID and chebi ID returned by kegg.
+ """
+ logger.info(f"KEGG: searching {entity_name} directly")
+ try:
+ kegg_response = httpx.get(f"{API_KEGG}/find/compound/{entity_name}")
+ if kegg_response is None or kegg_response.status_code != 200:
+ logger.warning(f"KEGG: Failed to retrieve KEGG ID for {entity_name}")
+ return None, None
+ kegg_text = kegg_response.text.strip()
+ if not kegg_text:
+ logger.warning(f"KEGG: No KEGG entry found for {entity_name}")
+ return None, None
+ kegg_id = kegg_text.split("\t")[0].strip()
+ kegg_id = kegg_id.replace("cpd:", "").split(";")[0].strip()
+ logger.info(f"KEGG: Extracted KEGG ID {kegg_id}")
+
+ logger.info(f"KEGG: Converting KEGG ID {kegg_id} to a pubchem id")
+ try:
+ pubchem_response = httpx.get(f"{API_KEGG}/conv/pubchem/cpd:{kegg_id}")
+ if pubchem_response is None or pubchem_response.status_code != 200:
+ logger.warning(f"KEGG: Failed to retrieve PubChem ID for {entity_name}")
+ return None, None
+ pubchem_parts = pubchem_response.text.strip().split("\t")
+ logger.info(f"KEGG: Extracted Pubchem parts : {pubchem_parts}")
+ if len(pubchem_parts) < 2:
+ logger.warning(f"KEGG: No PubChem mapping in KEGG for {entity_name}")
+ return None, None
+ pubchem_id_from_kegg = (
+ pubchem_parts[1].split("\n")[0].replace("pubchem:", "").strip()
+ )
+ logger.info(
+ f"KEGG: Converted KEGG ID {kegg_id} to Pubchem {pubchem_id_from_kegg}"
+ )
+ logger.info(f"KEGG: Converting KEGG ID {kegg_id} to ChEBI ID")
+ try:
+ chebi_response = httpx.get(f"{API_KEGG}/conv/chebi/cpd:{kegg_id}")
+ if chebi_response is None or chebi_response.status_code != 200:
+ logger.warning(
+ f"KEGG: Failed to retrieve CHEBI ID for {entity_name}"
+ )
+ return None, None
+ chebi_parts = chebi_response.text.strip().split("\t")
+ logger.info(f"KEGG: Extracted ChEBI parts : {chebi_parts}")
+ if len(chebi_parts) < 2:
+ logger.warning(f"KEGG: No ChEBI mapping in KEGG for {entity_name}")
+ return pubchem_id_from_kegg, None
+ chebi_id_from_kegg = (
+ chebi_parts[1].split("\n")[0].replace("chebi:", "").strip()
+ )
+ logger.info(
+ f"KEGG: Converted KEGG ID {kegg_id} to {chebi_id_from_kegg}"
+ )
+ return pubchem_id_from_kegg, chebi_id_from_kegg
+ except httpx.RemoteProtocolError as e:
+ logger.warning(
+ f"RemoteProtocolError on {API_KEGG} for {entity_name}: {e}"
+ )
+ return None, None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_KEGG} for {entity_name}: {e}")
+ return None, None
+
+
+def query_chebi_by_name(entity_name: str) -> str | None:
+ """Return the CHEBI ID linked to the molecule name.
+
+ Parameters
+ ----------
+ entity_name : str
+ The name of the molecule to query.
+
+ Returns
+ -------
+ str | None
+ The CHEBI ID returned by CHEBI.
+ """
+ logger.info(f"ChEBI: searching {entity_name}")
+ try:
+ chebi_response = httpx.get(API_CHEBI, params={"term": entity_name}, timeout=200)
+ if chebi_response is not None and chebi_response.status_code == 200:
+ results = chebi_response.json().get("results", [])
+ if not results:
+ logger.warning(f"ChEBI: No ChEBI entry found for {entity_name}")
+ return None
+ chebi_id = results[0]["_id"]
+ logger.info(f"ChEBI: Found CHEBI ID for {chebi_id}")
+ return chebi_id
+ logger.warning(f"ChEBI: Failed to retrieve CHEBI ID for {entity_name}")
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_CHEBI} for {entity_name}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_CHEBI} for {entity_name}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_CHEBI} for {entity_name}: {e}")
+ return None
+
+
+def query_pubchem_by_name(entity_name: str) -> str | None:
+ """Return the PubChem compound ID linked to the molecule name.
+
+ Parameters
+ ----------
+ entity_name : str
+ The name of the molecule to query.
+
+ Returns
+ -------
+ str | None
+ The PubChem compound ID returned by PubChem.
+ """
+ logger.info(f"Pubchem: searching {entity_name}")
+ try:
+ response = httpx.get(f"{API_PUBCHEM}/compound/name/{entity_name}/JSON")
+ if response is not None and response.status_code == 200:
+ compounds = response.json().get("PC_Compounds", [])
+ if not compounds:
+ logger.warning(f"PubChem: No compound found for {entity_name}")
+ return None
+ pubchem_id = str(compounds[0]["id"]["id"]["cid"])
+ logger.info(f"PubChem: Found Pubchem ID {pubchem_id}")
+ return pubchem_id
+ logger.warning(f"PubChem: Failed to retrieve PubChem ID for {entity_name}")
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_PUBCHEM} for {entity_name}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_PUBCHEM} for {entity_name}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_PUBCHEM} for {entity_name}: {e}")
+ return None
+
+
+def query_pubchem_by_substance(substance_id: str) -> str | None:
+ """Return the PubChem compound ID linked to the substance ID.
+
+ Parameters
+ ----------
+ substance_id : str
+ The substance ID to query.
+
+ Returns
+ -------
+ str | None
+ The PubChem compound ID returned by PubChem.
+ """
+ logger.info(f"Pubchem Substance: searching pubchem compound ID from {substance_id}")
+ try:
+ response = httpx.get(f"{API_PUBCHEM}/substance/sid/{substance_id}/cids/JSON")
+ if response is not None and response.status_code == 200:
+ information = (
+ response.json().get("InformationList", {}).get("Information", [])
+ )
+ if not information:
+ logger.warning(
+ f"Pubchem Substance: No CID found for substance {substance_id}"
+ )
+ return None
+ pubchem_id_from_substance = str(information[0]["CID"])
+ logger.info(
+ f"Pubchem Substance: Found Pubchem compound ID"
+ f"{pubchem_id_from_substance}"
+ )
+ return pubchem_id_from_substance
+ logger.warning(
+ f"Pubchem Substance: Failed to retrieve PubChem compound ID for"
+ f"{substance_id}"
+ )
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_PUBCHEM} for {substance_id}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_PUBCHEM} for {substance_id}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_PUBCHEM} for {substance_id}: {e}")
+ return None
+
+
+def query_pubchem_substance_by_name(entity_name: str) -> str | None:
+ """Return the PubChem substance ID linked to the molecule name.
+
+ Parameters
+ ----------
+ entity_name : str
+ The name of the molecule to query.
+
+ Returns
+ -------
+ str | None
+ The first PubChem substance ID found, or None.
+ """
+ logger.info(f"PubChem Substance: Searching {entity_name}")
+ try:
+ response = httpx.get(f"{API_PUBCHEM}/substance/name/{entity_name}/JSON")
+ if response is None or response.status_code != 200:
+ logger.warning(
+ f"PubChem Substance: Failed to retrieve PubChem substance for"
+ f"{entity_name}"
+ )
+ return None
+ substances = response.json().get("PC_Substances", [])
+ if not substances:
+ logger.warning(
+ f"PubChem Substance: No PubChem substance found for {entity_name}"
+ )
+ return None
+ sid = substances[0].get("sid", {}).get("id")
+ if sid is None:
+ logger.warning(
+ f"PubChem Substance: No SID found in PubChem substance response for"
+ f" {entity_name}"
+ )
+ return None
+ sid = str(sid)
+ logger.info(f"PubChem Substance: Found Pubchem SID {sid} for {entity_name}")
+ return sid
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_PUBCHEM} for {entity_name}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_PUBCHEM} for {entity_name}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_PUBCHEM} for {entity_name}: {e}")
+ return None
+
+
+def get_chebi_id_from_pubchem_synonyms(pubchem_id: str) -> str | None:
+ """Return the CHEBI ID linked to the PubChem compound ID via synonyms.
+
+ Parameters
+ ----------
+ pubchem_id : str
+ The PubChem compound ID to query.
+
+ Returns
+ -------
+ str | None
+ The CHEBI ID found in PubChem synonyms.
+ """
+ logger.info(f"Pubchem: searching ChEBI synonyms from {pubchem_id}")
+ try:
+ response = httpx.get(
+ f"{API_PUBCHEM}/compound/cid/{pubchem_id}/synonyms/JSON", timeout=200
+ )
+ if response is not None and response.status_code == 200:
+ synonyms = response.json()["InformationList"]["Information"][0]["Synonym"]
+ for synonym in synonyms:
+ if synonym.startswith("CHEBI:"):
+ chebi_id_synonyms = synonym.replace("CHEBI:", "")
+ logger.info(
+ f"Pubchem: Found ChEBI ID {chebi_id_synonyms} from {pubchem_id}"
+ )
+ return chebi_id_synonyms
+ logger.warning(
+ f"Pubchem: No CHEBI ID found in synonyms for PubChem ID {pubchem_id}"
+ )
+ return None
+ logger.warning(
+ f"Pubchem: Failed to retrieve synonyms for PubChem ID {pubchem_id}"
+ )
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_PUBCHEM} for {pubchem_id}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_PUBCHEM} for {pubchem_id}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_PUBCHEM} for {pubchem_id}: {e}")
+ return None
+
+
+def get_chebi_id_from_pubchem_synonyms_from_sid(sid: str) -> str | None:
+ """Return the CHEBI ID found in the synonyms of a PubChem substance SID.
+
+ Parameters
+ ----------
+ sid : str
+ The PubChem substance ID to query.
+
+ Returns
+ -------
+ str | None
+ The CHEBI ID found in the substance synonyms, or None.
+ """
+ logger.info(f"Pubchem substance: Searching ChEBI ID from {sid}")
+ try:
+ response = httpx.get(f"{API_PUBCHEM}/substance/sid/{sid}/JSON")
+ if response is None or response.status_code != 200:
+ logger.warning(
+ f"Pubchem substance: Failed to retrieve substance data for SID {sid}"
+ )
+ return None
+ substances = response.json().get("PC_Substances", [])
+ for substance in substances:
+ if substance.get("sid", {}).get("id") == int(sid):
+ synonyms = substance.get("synonyms", [])
+ for synonym in synonyms:
+ if synonym.startswith("CHEBI:"):
+ chebi_id = synonym.replace("CHEBI:", "")
+ logger.info(
+ f"Pubchem substance: Found ChEBI ID in pubchem substance"
+ f" synonyms of{chebi_id}"
+ )
+ return chebi_id
+ logger.warning(
+ f"Pubchem substance: No CHEBI ID found in synonyms for SID {sid}"
+ )
+ return None
+ logger.warning(f"Pubchem substance: SID {sid} not found in PubChem response")
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_CHEBI} for {sid}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_CHEBI} for {sid}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_CHEBI} for {sid}: {e}")
+ return None
+
+
+def get_pubchem_cid_from_substance(sid: str) -> str | None:
+ """Return the PubChem compound ID linked to a substance ID via the compound field.
+
+ Parameters
+ ----------
+ sid : str
+ The PubChem substance ID to query.
+
+ Returns
+ -------
+ str | None
+ The PubChem compound ID found in the substance's compound field.
+ """
+ logger.info(
+ f"Pubchem substance: Searching Pubchem compound ID from substance id : {sid} "
+ )
+ try:
+ response = httpx.get(f"{API_PUBCHEM}/substance/sid/{sid}/JSON", timeout=200)
+ if response is None or response.status_code != 200:
+ logger.warning(
+ f"Pubchem substance: Failed to retrieve substance data for SID {sid}"
+ )
+ return None
+
+ substances = response.json().get("PC_Substances", [])
+ for substance in substances:
+ if substance.get("sid", {}).get("id") == int(sid):
+ compound_list = substance.get("compound", [])
+ for entry in compound_list:
+ cid = entry.get("id", {}).get("id", {}).get("cid")
+ if cid is not None:
+ cid = str(cid)
+ logger.info(
+ f"Pubchem substance: Found CID {cid} from substance ID{sid}"
+ )
+ return cid
+ logger.warning(
+ f"Pubchem substance: No CID found in compound field for SID {sid}"
+ )
+ return None
+
+ logger.warning(f"Pubchem substance: SID {sid} not found in PubChem response")
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_PUBCHEM} for {sid}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_PUBCHEM} for {sid}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_PUBCHEM} for {sid}: {e}")
+ return None
+
+
+def get_synonym_from_pubchem_substance(sid: str) -> str | None:
+ """Return the first synonym of a PubChem substance.
+
+ Parameters
+ ----------
+ sid : str
+ The PubChem substance ID to query.
+
+ Returns
+ -------
+ str | None
+ The first synonym found for the substance.
+ """
+ logger.info(
+ f"Pubchem substance: Searching for the first synonym of pubchem substance {sid}"
+ )
+ try:
+ response = httpx.get(f"{API_PUBCHEM}/substance/sid/{sid}/JSON")
+ if response is None or response.status_code != 200:
+ logger.warning(
+ f"Pubchem substance: Failed to retrieve substance data for SID {sid}"
+ )
+ return None
+
+ substances = response.json().get("PC_Substances", [])
+ for substance in substances:
+ if substance.get("sid", {}).get("id") == int(sid):
+ synonyms = substance.get("synonyms", [])
+ if not synonyms:
+ logger.warning(
+ f"Pubchem substance: No synonyms found for SID {sid}"
+ )
+ return None
+ logger.info(
+ f"Pubchem substance: Found {synonyms[0]} as {sid} first synonym "
+ )
+ return synonyms[0]
+
+ logger.warning(f"Pubchem substance: SID {sid} not found in PubChem response")
+ return None
+ except httpx.RemoteProtocolError as e:
+ logger.warning(f"RemoteProtocolError on {API_PUBCHEM} for {sid}: {e}")
+ return None
+ except httpx.TimeoutException as e:
+ logger.warning(f"Timeout on {API_PUBCHEM} for {sid}: {e}")
+ return None
+ except httpx.RequestError as e:
+ logger.warning(f"Request error on {API_PUBCHEM} for {sid}: {e}")
+ return None
+
+
+def get_compound_id_from_kegg_substance(sid: str) -> str | None:
+ """Return the PubChem compound ID from a KEGG substance SID using fallbacks.
+
+ First tries to find a CID directly in the substance's compound field.
+ If not found, tries the direct SID to CID mapping via query_pubchem_by_substance.
+ If still not found, takes the first synonym and searches PubChem compound by name.
+
+ Parameters
+ ----------
+ sid : str
+ The PubChem substance ID retrieved from KEGG.
+
+ Returns
+ -------
+ str | None
+ The PubChem compound ID found, or None if all methods fail.
+ """
+ cid = get_pubchem_cid_from_substance(sid)
+ logger.info(f"Lookig for pubchem compound ID from pubchem substance {sid}")
+
+ if cid is not None:
+ logger.info(f"CID {cid} found in compound field for SID {sid}")
+ return cid
+ logger.warning("None Pubchem compound ID found")
+ logger.info(f"Retry: Lookig for pubchem compound ID from pubchem substance {sid}")
+ cid = query_pubchem_by_substance(sid)
+ if cid is not None:
+ logger.info(f"CID {cid} found via direct SID mapping for SID {sid}")
+ return cid
+ logger.info(f"Retry: Searching synonyms of substance {sid}")
+ synonym = get_synonym_from_pubchem_substance(sid)
+ if synonym is None:
+ logger.warning(f"No synonym found fallback possible for SID {sid}")
+ return None
+ logger.success(f"Found first synonym '{synonym}' for SID {sid}")
+ logger.info(f"Retry: Searching pubchem ID for : {synonym} ")
+ cid = query_pubchem_by_name(synonym)
+ if cid is not None:
+ logger.success(f"CID {cid} found via synonym '{synonym}' for SID {sid}")
+ return cid
+
+ logger.warning(f"All methods failed to find a CID for SID {sid}")
+ return None
+
+
+def _chebi_from_pubchem_cid(pubchem_id: str) -> str | None:
+ """Return the ChEBI ID from a PubChem CID via compound synonyms.
+
+ Parameters
+ ----------
+ pubchem_id : str
+ The PubChem compound ID to query.
+
+ Returns
+ -------
+ str | None
+ The ChEBI ID found in synonyms, or None.
+ """
+ return get_chebi_id_from_pubchem_synonyms(pubchem_id)
+
+
+def _chebi_via_pubchem_substance(mol: str) -> tuple[str | None, str | None]:
+ """Fallback PubChem Substance path when no direct CID is available.
+
+ Attempts in order:
+ 1. Finds a SID from the molecule name searches SID synonyms for CHEBI:XXX.
+ 2. If no ChEBI in SID synonyms, resolves SID →CID via:
+ a. compound field of the SID
+ b. /substance/sid/{sid}/cids endpoint
+ c. first synonym of the SID compound name search
+ Returns (chebi_id_found, pubchem_cid_resolved).
+
+ Parameters
+ ----------
+ mol : str
+ The molecule name to query.
+
+ Returns
+ -------
+ tuple[str | None, str | None]
+ (chebi_id, resolved_pubchem_cid) either may be None.
+ """
+ logger.info(f"looking for pubchem substance for {mol}")
+ sid = query_pubchem_substance_by_name(mol)
+ if sid is None:
+ logger.warning(f"No SID PubChem found for {mol}")
+ return None, None
+
+ logger.success(f"Found pubchem substance ID: {sid} for {mol}")
+
+ logger.info("looking for ChEBI ID in substance synonyms")
+ chebi_from_sid = get_chebi_id_from_pubchem_synonyms_from_sid(sid)
+ if chebi_from_sid is not None:
+ logger.success(
+ f"Found ChEBI ID: {chebi_from_sid} in SID synonyms: {chebi_from_sid}"
+ )
+ return chebi_from_sid, None
+
+ logger.warning("No ChEBI ID found in SID synonymes")
+ logger.info("Trying to convert SID to CID ")
+ cid = get_compound_id_from_kegg_substance(sid)
+ if cid is None:
+ logger.warning(f"No Pubchem compound found from {sid}")
+ return None, None
+
+ logger.success(f"Found compound ID from substance ID: {cid}")
+ logger.info(f"Searching for ChEBI ID from CID {cid}")
+ chebi_from_cid = _chebi_from_pubchem_cid(cid)
+ return chebi_from_cid, cid
+
+
+def get_chebi_id_for_molecule(mol: str) -> tuple[str | None, str | None, str | None]:
+ """Resolve ChEBI IDs for a molecule from ChEBI, PubChem, and KEGG.
+
+ Covers the following paths:
+ - Direct ChEBI search by name.
+ - PubChem Compound: name then CID then compound synonyms for CHEBI:XXX.
+ - PubChem Substance fallback (when no CID): name then SID then SID synonyms,
+ then SID then CID (compound field / cids endpoint / first synonym) then
+ compound synonyms.
+ - KEGG: name then KEGG ChEBI ID directly.
+ - KEGG PubChem CID (if different from direct): CID then compound synonyms.
+ - KEGG SID fallback: if KEGG returns a substance ID, resolve to CID then synonyms.
+
+ Parameters
+ ----------
+ mol : str
+ The molecule entity name.
+
+ Returns
+ -------
+ tuple[str | None, str | None, str | None]
+ (chebi_id_direct, chebi_id_from_pubchem, chebi_id_from_kegg)
+ """
+ logger.info("Looking for direct Chebi and Pubchem ID")
+ chebi_id = query_chebi_by_name(mol)
+ logger.info(f"direct ChEBI ID: {chebi_id!r}")
+
+ pubchem_cid = query_pubchem_by_name(mol)
+ logger.info(f"direct PubChem CID: {pubchem_cid!r}")
+
+ if pubchem_cid is not None:
+ chebi_from_pubchem = _chebi_from_pubchem_cid(pubchem_cid)
+ logger.success(f"Found ChEBI via CID ({pubchem_cid}): {chebi_from_pubchem!r}")
+ else:
+ logger.warning("No pubchem CID found retrying via Substance...")
+ chebi_from_pubchem, pubchem_cid = _chebi_via_pubchem_substance(mol)
+ logger.info(f"ChEBI via Substance: {chebi_from_pubchem}")
+
+ pubchem_id_from_kegg, chebi_from_kegg = query_kegg_by_name(mol)
+ logger.info(
+ f" → PubChem via KEGG: {pubchem_id_from_kegg!r}, ChEBI via KEGG: "
+ f"{chebi_from_kegg!r}"
+ )
+
+ if pubchem_id_from_kegg is not None and pubchem_id_from_kegg != pubchem_cid:
+ logger.info(
+ "Different CID between KEGG and direct pubchem , loonking in ChEBI with "
+ "this CID..."
+ )
+ chebi_via_kegg_cid = _chebi_from_pubchem_cid(pubchem_id_from_kegg)
+ if chebi_via_kegg_cid is not None:
+ logger.info(f"ChEBI via CID KEGG: {chebi_via_kegg_cid!r}")
+ chebi_from_kegg = chebi_via_kegg_cid
+ else:
+ logger.info(
+ f"No ChEBI via CID KEGG, searching for compound from SID KEGG: "
+ f"{pubchem_id_from_kegg}"
+ )
+ resolved_cid = get_compound_id_from_kegg_substance(pubchem_id_from_kegg)
+ if resolved_cid is not None:
+ chebi_from_kegg = _chebi_from_pubchem_cid(resolved_cid)
+ logger.success(
+ f"Found ChEBI ID via SID KEGG {resolved_cid}: {chebi_from_kegg}"
+ )
+
+ return chebi_id, chebi_from_pubchem, chebi_from_kegg
+
+
+def compare_chebi_ids(
+ chebi_id: str | None,
+ chebi_id_from_kegg: str | None,
+ chebi_id_from_pubchem: str | None,
+ mol: str,
+) -> bool:
+ """Compare CHEBI IDs from ChEBI, KEGG and PubChem.
+
+ Parameters
+ ----------
+ chebi_id : str | None
+ The CHEBI ID retrieved directly from ChEBI.
+ chebi_id_from_kegg : str | None
+ The CHEBI ID retrieved via KEGG.
+ chebi_id_from_pubchem : str | None
+ The CHEBI ID retrieved via PubChem synonyms.
+ mol : str
+ The molecule entity name (used for logging).
+
+ Returns
+ -------
+ bool
+ True if at least 2 CHEBI IDs match, False otherwise.
+ """
+ if (
+ (
+ chebi_id is not None
+ and chebi_id_from_pubchem is not None
+ and chebi_id == chebi_id_from_pubchem
+ )
+ or (
+ chebi_id is not None
+ and chebi_id_from_kegg is not None
+ and chebi_id == chebi_id_from_kegg
+ )
+ or (
+ chebi_id_from_pubchem is not None
+ and chebi_id_from_kegg is not None
+ and chebi_id_from_pubchem == chebi_id_from_kegg
+ )
+ ):
+ logger.info(f"CHEBI ID for {mol} is the same in at least 2 databases.")
+ return True
+ logger.warning(f"CHEBI ID for {mol} is different across databases.")
+ return False
+
+
+def compare_pubchem_ids(
+ pubchem_id: str | None,
+ pubchem_id_from_kegg: str | None,
+ mol: str,
+) -> bool:
+ """Compare PubChem IDs from direct query and KEGG.
+
+ Parameters
+ ----------
+ pubchem_id : str | None
+ The PubChem ID retrieved directly from PubChem.
+ pubchem_id_from_kegg : str | None
+ The PubChem ID retrieved via KEGG.
+ mol : str
+ The molecule entity name (used for logging).
+
+ Returns
+ -------
+ bool
+ True if the PubChem IDs match, False otherwise.
+ """
+ if pubchem_id is None or pubchem_id_from_kegg is None:
+ logger.warning(f"PubChem ID missing for {mol}, cannot compare.")
+ return False
+
+ if pubchem_id == pubchem_id_from_kegg:
+ logger.info(f"PubChem ID for {mol} matches directly.")
+ return True
+
+ pubchem_id_from_kegg_compound = get_compound_id_from_kegg_substance(
+ pubchem_id_from_kegg
+ )
+ if (
+ pubchem_id_from_kegg_compound is not None
+ and pubchem_id_from_kegg_compound == pubchem_id
+ ):
+ logger.info(f"PubChem ID for {mol} matches after resolving KEGG substance.")
+ return True
+
+ logger.warning(f"PubChem ID for {mol} does not match across sources.")
+ return False
+
+
+def save_chebi_comparaison_in_tsv(
+ molecules: list[str],
+ output_file: Path,
+) -> None:
+ """Save the ChEBI ID comparison results in a TSV file.
+
+ Parameters
+ ----------
+ molecules : list[str]
+ The list of molecule entity names.
+ output_file : Path
+ The path to the output TSV file.
+ """
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+
+ with open(output_file, "w", encoding="utf-8") as file:
+ file.write(
+ "Molecule\tCHEBI_ID\tCHEBI_ID_from_KEGG\tCHEBI_ID_from_PubChem\tMatch\n"
+ )
+
+ for index, mol in enumerate(molecules, start=1):
+ logger.info("=" * 50)
+ logger.info(f"Processing molecule {index}/{len(molecules)}: '{mol}'")
+
+ chebi_id, chebi_from_pubchem, chebi_from_kegg = get_chebi_id_for_molecule(
+ mol
+ )
+
+ match = compare_chebi_ids(
+ chebi_id, chebi_from_kegg, chebi_from_pubchem, mol
+ )
+ logger.info(f" → Match (au moins 2 IDs identiques): {match}")
+
+ line = (
+ f"{mol}\t{chebi_id}\t{chebi_from_kegg}\t{chebi_from_pubchem}\t{match}\n"
+ )
+ file.write(line)
+
+ logger.info("=" * 50)
+ logger.info(f"Fichier sauvegardé: {output_file}")
+
+
+def get_no_chebi_match(chebi_comparaison_file: Path) -> list[str]:
+ """Extract molecule names with no CHEBI ID match from the comparison file.
+
+ Parameters
+ ----------
+ chebi_comparaison_file : Path
+ The path to the CHEBI comparison TSV file.
+
+ Returns
+ -------
+ list[str]
+ A list of molecule names that have no CHEBI ID match across databases.
+ """
+ df = pd.read_csv(chebi_comparaison_file, sep="\t")
+ no_match_df = df[df["Match"] == False]
+ return no_match_df["Molecule"].tolist()
+
+
+def save_pubchem_comparaison_in_tsv(
+ molecules: list[str],
+ output_file: Path,
+) -> None:
+ """Save the PubChem ID comparison results in a TSV file.
+
+ Parameters
+ ----------
+ molecules : list[str]
+ The list of molecule entity names.
+ output_file : Path
+ The path to the output TSV file.
+ """
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+
+ with open(output_file, "w") as file:
+ file.write("Molecule\tPubChem_ID\tPubChem_ID_from_KEGG\tMatch\n")
+ for index, mol in enumerate(molecules, start=1):
+ logger.info(f"Processing molecule {index}/{len(molecules)}: {mol}")
+ pubchem_id = query_pubchem_by_name(mol)
+ pubchem_id_from_kegg, _ = query_kegg_by_name(mol)
+
+ match = compare_pubchem_ids(pubchem_id, pubchem_id_from_kegg, mol)
+
+ file.write(f"{mol}\t{pubchem_id}\t{pubchem_id_from_kegg}\t{match}\n")
+
+
+def load_molecule_entities(file_path: Path) -> list:
+ """Load molecular identifiers from a file into a list.
+
+ Parameters
+ ----------
+ file_path : Path
+ Path to the input file containing molecular identifiers.
+
+ Returns
+ -------
+ list
+ A list of molecular identifiers loaded from the file.
+ """
+ logger.info(f"Loading MOL entities from {file_path}...")
+ entities = pd.read_csv(file_path, sep="\t")
+ mol_entities = entities[entities["category"] == "MOL"]
+ mol_entities = list(mol_entities["entity"].unique())
+ molecule_liste = []
+ for molecule in mol_entities:
+ if len(molecule) > 3:
+ molecule_liste.append(molecule)
+ molecule_liste = list(set(molecule_liste))
+ logger.info(f"Loaded {len(molecule_liste)} MOL entities successfully.")
+ return molecule_liste
+
+
+if __name__ == "__main__":
+ entities_file = Path("data/entities.tsv")
+ output_dir = Path("results/ground_molecule/same_grounding_mol")
+
+ all_molecules = load_molecule_entities(entities_file)
+
+ small_molecules, other_entities = filter_molecules(all_molecules)
+
+ # save_pdb_uniprot_seq_entities(
+ # other_entities=other_entities,
+ # output_file=output_dir / "pdb_uniprot_seq_entities.tsv",
+ # )
+
+ chebi_output = output_dir / "chebi_comparaison.tsv"
+ # save_chebi_comparaison_in_tsv(
+ # molecules=small_molecules,
+ # output_file=chebi_output,
+ # )
+
+ # Pipeline PubChem pour les molécules sans match ChEBI
+ no_chebi_match_molecules = get_no_chebi_match(
+ Path("results/ground_molecule/same_grounding_mol/chebi_comparaison.tsv")
+ )
+ logger.info(
+ f"Number of molecules with no CHEBI ID match: {len(no_chebi_match_molecules)}"
+ )
+ logger.info("Saving PubChem comparison for molecules with no CHEBI match...")
+ save_pubchem_comparaison_in_tsv(
+ molecules=no_chebi_match_molecules,
+ output_file=output_dir / "pubchem_comparaison_no_chebi_match.tsv",
+ )
diff --git a/src/mdner_llm/core/normalize_entities.py b/src/mdner_llm/normalization/normalize_entities.py
similarity index 63%
rename from src/mdner_llm/core/normalize_entities.py
rename to src/mdner_llm/normalization/normalize_entities.py
index e21e7b7..5d495a0 100644
--- a/src/mdner_llm/core/normalize_entities.py
+++ b/src/mdner_llm/normalization/normalize_entities.py
@@ -13,7 +13,10 @@
from pydantic import ValidationError
from mdner_llm.logger import create_logger
-from mdner_llm.models.entities import ListOfEntities, ListOfEntitiesNormalized
+from mdner_llm.models.entities import ListOfEntities
+from mdner_llm.models.entities_normalized import ListOfEntitiesNormalized
+from mdner_llm.normalization.normalize_stemp import norm_temp
+from mdner_llm.normalization.normalize_stime_with_regex import norm_stime_regex
STIME_RE = re.compile(r"([0-9]+)(\.?[0-9]+)? *(ps|ns|μs|ms|s)", re.IGNORECASE)
STEMP_RE = re.compile(r"([0-9]+)(\.?[0-9]+)?( *˚? *[a-z]*)?", re.IGNORECASE)
@@ -130,8 +133,90 @@ def check_hallucination(
return True
+def norm_ffm(ffm_db: dict[str, Any], predicted_entity: str) -> dict[str, Any]:
+ """Normalize a force field model entity using the provided database entry.
+
+ Returns
+ -------
+ dict[str, Any]: A dictionary containing normalized fields
+ for the force field model.
+ """
+ entry = {}
+ for ffm in ffm_db:
+ # Create a list of names and aliases for comparison
+ names = [ffm.get("name", ""), *ffm.get("aliases", [])]
+ if predicted_entity in [name.lower().strip() for name in names]:
+ entry = ffm
+ break
+ if not entry:
+ logger.warning(f"FFM: No match found for '{predicted_entity}'.")
+ return {
+ "text_normalized": entry.get("name") or predicted_entity,
+ "tag": entry.get("category"),
+ "family": entry.get("family"),
+ "aliases": entry.get("aliases"),
+ "resolution": entry.get("resolution"),
+ "molecular_type": entry.get("molecular_type"),
+ "ontology_link": entry.get("ontology_link"),
+ "publication_link": entry.get("publication"),
+ }
+
+
+def norm_softname(predicted_entity: str, base_dir: Path) -> dict[str, Any]:
+ """Normalize a software name entity using its local codemeta.json file.
+
+ Returns
+ -------
+ dict[str, Any]: A dictionary containing normalized fields
+ for the software name, or defaults if not found.
+ """
+ # Path to the expected codemeta.json file
+ meta_path = base_dir / predicted_entity / "codemeta.json"
+ # Load data if file exists, otherwise use an empty dictionary
+ meta = {}
+ if meta_path.exists():
+ try:
+ with open(meta_path, encoding="utf-8") as json_file:
+ meta = json.load(json_file)
+ except json.JSONDecodeError:
+ logger.warning(
+ f"SOFTNAME: Failed to parse codemeta.json for '{predicted_entity}'."
+ )
+ else:
+ logger.warning(f"SOFTNAME: codemeta.json not found for '{predicted_entity}'.")
+ # Map CodeMeta keys to Pydantic model attributes
+ raw_authors = meta.get("author", [])
+ formatted_authors = [
+ {
+ "id": a.get("id"),
+ "type": a.get("type", "Person"),
+ "first_name": a.get("givenName", "").strip(),
+ "last_name": a.get("familyName", "").strip(),
+ "affiliation": a.get("affiliation"),
+ }
+ for a in raw_authors
+ ]
+ return {
+ "name": meta.get("name", predicted_entity).strip(),
+ "authors": formatted_authors,
+ "description": meta.get("description"),
+ "version": meta.get("version"),
+ "date_last_modification": meta.get("dateModified"),
+ "code_repository_link": meta.get("codeRepository"),
+ "download_url": meta.get("downloadUrl"),
+ "related_link": meta.get("relatedLink"),
+ "publication_link": meta.get("referencePublication"),
+ "license": meta.get("license"),
+ "keywords": meta.get("keywords"),
+ "programming_language": meta.get("programmingLanguage"),
+ }
+
+
def normalize_json_content(
- data: dict[str, Any], logger: "loguru.Logger" = loguru.logger
+ data: dict[str, Any],
+ ffm_db: dict[str, Any],
+ softname_codemeta_dir: Path,
+ logger: "loguru.Logger" = loguru.logger,
) -> dict[str, Any] | None:
"""Normalize entities in the JSON content and check for hallucinations.
@@ -162,6 +247,18 @@ def normalize_json_content(
data.get("input_json_path", "NA"),
data.get("url", "NA"),
)
+ if entity.category == "STEMP":
+ ent_dict["value"], ent_dict["unit"] = norm_temp(entity.text)
+ elif entity.category == "FFM":
+ ent_dict.update(norm_ffm(ffm_db, predicted_entity_cleaned))
+ elif entity.category == "SOFTNAME":
+ ent_dict.update(
+ norm_softname(predicted_entity_cleaned, softname_codemeta_dir)
+ )
+ elif entity.category == "STIME":
+ ent_dict.update(norm_stime_regex(predicted_entity_cleaned))
+ elif entity.category == "MOL":
+ pass
normalized_entities.append(ent_dict)
# Create a new ListOfEntitiesNormalized instance and validate it
try:
@@ -200,12 +297,24 @@ def save_json_data(
type=click.Path(exists=True, file_okay=False, path_type=Path),
help="Directory containing the input inference JSON files.",
)
+@click.option(
+ "--ffm-db-path",
+ type=click.Path(exists=True, file_okay=True, path_type=Path),
+ help="Path to the force field database JSON file.",
+)
+@click.option(
+ "--softname-codemeta-dir",
+ type=click.Path(exists=True, file_okay=False, path_type=Path),
+ help="Directory containing the software name codemeta files.",
+)
@click.option(
"--output-dir",
type=click.Path(file_okay=False, path_type=Path),
help="Directory where normalized JSON files will be saved.",
)
-def main(input_dir: Path, output_dir: Path) -> None:
+def main(
+ input_dir: Path, ffm_db_path: Path, softname_codemeta_dir: Path, output_dir: Path
+) -> None:
"""Load JSON files, normalize their entities, and save the updated data."""
logger = create_logger(f"logs/normalize_{input_dir.name}.log")
# Create the output directory if it doesn't exist
@@ -218,14 +327,26 @@ def main(input_dir: Path, output_dir: Path) -> None:
return
logger.info(f"Found {total_files} JSON file(s) to process from {input_dir}.")
+ # Load the force field database for normalization
+ try:
+ with open(ffm_db_path, encoding="utf-8") as f:
+ json_data = json.load(f)
+ ffm_db = json_data.get("force_fields", [])
+ logger.info(f"Loaded force field database from {ffm_db_path}.")
+ except (FileNotFoundError, JSONDecodeError) as e:
+ logger.error(f"Failed to load force field database: {e}")
+ return
+
processed_count = 0
for file_path in json_files:
- # Load the json file
+ # Load the json inference file
data = load_json_data(file_path)
if data is None:
continue
# Normalize the entities in the JSON content
- updated_data = normalize_json_content(data, logger)
+ updated_data = normalize_json_content(
+ data, ffm_db, softname_codemeta_dir, logger
+ )
if updated_data is None:
continue
# Save the updated data to the output directory
diff --git a/src/mdner_llm/normalization/normalize_stemp.py b/src/mdner_llm/normalization/normalize_stemp.py
new file mode 100644
index 0000000..836c4b4
--- /dev/null
+++ b/src/mdner_llm/normalization/normalize_stemp.py
@@ -0,0 +1,120 @@
+"""Script to normalize simulation temperature entities."""
+
+import re
+from pathlib import Path
+
+import click
+import pandas as pd
+from loguru import logger
+
+
+def norm_temp(temp_str: str) -> tuple:
+ """Normalize the value of a temperature.
+
+ Returns
+ -------
+ str: The normalized value of the temp_str
+ """
+ temp_str = temp_str.lower() # We convert the temp_str to lowercase
+ # Extraction of the temperature and unit with a regex and the
+ # search method of the re module
+ # The regex is composed of three groups:
+ # - The first group matches the integer part:([0-9])+ allows to math one or more
+ # digit due to the "+" symbol
+ # - The second group matches the decimal part: (\.?[0-9]+)? allows to match
+ # an optional decimal part due to the "?" symbol at the end of the group.
+ # This group consists of an optional dot due to the "\.?" but if present is
+ # necessary followed by one or more digits thanks to the "[0-9]+" part.
+ # - The third group matches the unit: ( *°? *[a-z]*)? allows to match
+ # an optional unit because of the "?" symbol at the end of the group.
+ # This group consists of zero or more spaces, because of the "*" symbol,
+ # an optional degree symbol, then zero or more spaces, and zero or more letters.
+ # logger.info("Normalising temperature entities...")
+ # If the temperatue is anotated as room temperature or body temperature
+ # we normalize it to the standard value
+ if temp_str == "room temperature":
+ return (293, "K")
+ if temp_str == "human body temperature":
+ return (310, "K")
+ temperature_match = re.search(r"([0-9]+)(\.?[0-9]+)?( *˚? *[a-z]*)?", temp_str)
+ if temperature_match is None:
+ logger.warning(f"STEMP: Could not extract temperature from '{temp_str}'.")
+ return None, None
+ # logger.info("Found temperature entity...")
+ temperature_integer_part = temperature_match.group(1)
+ temperature_decimal_part = temperature_match.group(2)
+ temperature_unit = temperature_match.group(3)
+ # Fetching the temperature value and casting to int or float
+ if temperature_decimal_part is not None:
+ temperature_value = float(
+ temperature_integer_part + temperature_decimal_part.strip()
+ )
+ else:
+ temperature_value = int(temperature_integer_part)
+
+ # Fetching the unit and converting to kelvin when needed
+ if temperature_unit is not None:
+ temperature_unit = temperature_unit.strip(" ") # We remove the spaces
+ temperature_unit = temperature_unit.strip("°") # We remove the degree
+ if temperature_unit == "": # if there is no unit we assume it's kelvin
+ temperature_unit = "K"
+ elif "c" in temperature_unit:
+ # if the unit is in celsius we convert it to kelvin
+ temperature_value += 273.15
+ temperature_unit = "K"
+
+ return temperature_value, temperature_unit
+
+
+def create_norm_temp_file(raw_temp_file: Path, norm_temp_file: Path) -> None:
+ """Create a .tsv file containing the raw temperature value.
+
+ the normalised temperature value and the normalised unit.
+
+ Parameters
+ ----------
+ raw_temp_file (Path) : path to the input file containing the raw values
+ norm_temp_file (Path) : path to the input file with the normalised informations
+
+ """
+ df = pd.read_csv(raw_temp_file, sep="\t")
+ temp_entities = df[df["category"] == "STEMP"]["entity"].tolist()
+
+ if not norm_temp_file.parent.exists():
+ norm_temp_file.parent.mkdir(parents=True, exist_ok=True)
+
+ with open(norm_temp_file, "w") as f2:
+ f2.write(
+ "raw_temperature\tnormalised_temperature\tnormalised_unit\tnormalized_result\n"
+ )
+
+ for raw_temp in temp_entities:
+ temperature_value, temperature_unit = norm_temp(raw_temp)
+
+ if temperature_value is not None:
+ f2.write(
+ f"{raw_temp}\t{temperature_value}\t{temperature_unit}"
+ f"\t{str(temperature_value) + temperature_unit}\n"
+ )
+ else:
+ f2.write(f"{raw_temp}\tERROR\tERROR\tERROR\n")
+
+
+@click.command()
+@click.option(
+ "--raw-entities-path",
+ type=click.Path(exists=True, dir_okay=False, path_type=Path),
+ help="Path to the input file containing raw temperature entities.",
+)
+@click.option(
+ "--normalized-stemp-path",
+ type=click.Path(dir_okay=False, path_type=Path),
+ help="Path to the output file for normalized temperature entities.",
+)
+def main(raw_entities_path: Path, normalized_stemp_path: Path):
+ """Normalize all the temperature entities in the input file and visualization."""
+ create_norm_temp_file(raw_entities_path, normalized_stemp_path)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/mdner_llm/normalization/normalize_stime_with_regex.py b/src/mdner_llm/normalization/normalize_stime_with_regex.py
new file mode 100644
index 0000000..f33d0db
--- /dev/null
+++ b/src/mdner_llm/normalization/normalize_stime_with_regex.py
@@ -0,0 +1,133 @@
+"""Script to get regex normalisation results."""
+
+import re
+from pathlib import Path
+
+import click
+import pandas as pd
+from loguru import logger
+
+PATTERN = re.compile(
+ r"([0-9]+)(\.?[0-9]+)? *(ps|ns|μs|ms|s)",
+ re.IGNORECASE,
+)
+
+UNITS_TO_NS = {
+ "ps": 1e-3,
+ "ns": 1,
+ "μs": 1e3,
+ "us": 1e3,
+ "ms": 1e6,
+ "s": 1e9,
+}
+
+
+def get_stime_entities(entities_file: Path) -> pd.DataFrame:
+ """Load entities from TSV and filter for STIME category.
+
+ Parameters
+ ----------
+ entities_file (Path): Path to the TSV file containing entities.
+
+ Returns
+ -------
+ pd.DataFrame: DataFrame containing only STIME entities.
+ """
+ entities = pd.read_csv(entities_file, sep="\t")
+ stime_entities = entities[entities["category"] == "STIME"].copy()
+ return stime_entities
+
+
+def norm_stime_regex(stime_entity: str) -> dict:
+ """Normalize a single simulation time entity using regex pattern matching.
+
+ Parameters
+ ----------
+ stime_entity (str): The raw simulation time string to normalize.
+
+ Returns
+ -------
+ dict: A dictionary with keys 'STIME', 'regex_value', and 'regex_unit'.
+ """
+ reg_value, reg_unit = None, None
+ match = PATTERN.search(str(stime_entity))
+ if match:
+ reg_value = float(match.group(1) + (match.group(2) or ""))
+ reg_unit = match.group(3).strip().lower()
+ if reg_unit not in UNITS_TO_NS:
+ reg_value, reg_unit = None, None
+ else:
+ logger.warning(f"STIME: No regex match found for '{stime_entity}'.")
+ return {
+ "STIME": str(stime_entity),
+ "regex_value": reg_value if reg_value is not None else "None",
+ "regex_unit": reg_unit if reg_unit is not None else "None",
+ }
+
+
+def normalize_stim_regex(stime_entities: pd.DataFrame) -> list[dict]:
+ """Normalize simulation time entities using regex pattern matching.
+
+ Parameters
+ ----------
+ stime_entities (pd.DataFrame): DataFrame containing STIME entities.
+
+ Returns
+ -------
+ list[dict]: List of dicts with keys STIME, regex_value, and regex_unit.
+ """
+ results = []
+ for _, row in stime_entities.iterrows():
+ reg_value, reg_unit = None, None
+ match = PATTERN.search(str(row["entity"]))
+ if match:
+ reg_value = float(match.group(1) + (match.group(2) or ""))
+ reg_unit = match.group(3).strip().lower()
+ if reg_unit not in UNITS_TO_NS:
+ reg_value, reg_unit = None, None
+ results.append(
+ {
+ "STIME": str(row["entity"]),
+ "regex_value": reg_value if reg_value is not None else "None",
+ "regex_unit": reg_unit if reg_unit is not None else "None",
+ }
+ )
+ return results
+
+
+def save_results_to_tsv(results: list, output_file: Path):
+ """Save the LLM normalization results to a TSV file.
+
+ Parameters
+ ----------
+ results: A list of dictionaries containing the normalization results.
+ output_file: Path to the output TSV file.
+ """
+ results_df = pd.DataFrame(results)
+ results_df_sorted = results_df.sort_values(by="STIME")
+ results_df_sorted.to_csv(output_file, sep="\t", index=False)
+ print(f"\nFile TSV saved : {output_file} ({len(results_df_sorted)} lines)")
+
+
+@click.command()
+@click.option(
+ "--entities-file",
+ type=click.Path(exists=True, dir_okay=False, path_type=Path),
+ default=Path("data/entities.tsv"),
+ help="Path to the TSV file containing the entities.",
+)
+@click.option(
+ "--output-file",
+ type=click.Path(dir_okay=False, path_type=Path),
+ default=Path("results/norm_simu_times/normalized_stime_results.tsv"),
+ help="Path to the output TSV file for normalized results.",
+)
+def main(entities_file: Path, output_file: Path):
+ """Run regex-based normalization of simulation time entities and save results."""
+ stime_entities = get_stime_entities(entities_file)
+ results = normalize_stim_regex(stime_entities)
+ save_results_to_tsv(results, output_file)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/mdner_llm/normalization/normalize_stime_wth_llm.py b/src/mdner_llm/normalization/normalize_stime_wth_llm.py
new file mode 100644
index 0000000..6c7d7f2
--- /dev/null
+++ b/src/mdner_llm/normalization/normalize_stime_wth_llm.py
@@ -0,0 +1,167 @@
+"""Script to get llm normalisation results."""
+
+import json
+from pathlib import Path
+
+import click
+import pandas as pd
+from mdverse_entity_norm.scripts.evaluate_llm_models import (
+ normalize_simulation_time,
+)
+
+UNITS_TO_NS = {
+ "ps": 1e-3,
+ "ns": 1,
+ "μs": 1e3,
+ "us": 1e3,
+ "ms": 1e6,
+ "s": 1e9,
+}
+
+
+def get_stime_entities(entities_file: Path) -> pd.DataFrame:
+ """Load entities from TSV and filter for STIME category.
+
+ Parameters
+ ----------
+ entities_file (Path): Path to the TSV file containing entities.
+
+ Returns
+ -------
+ pd.DataFrame: DataFrame containing only STIME entities.
+ """
+ entities = pd.read_csv(entities_file, sep="\t")
+ stime_entities = entities[entities["category"] == "STIME"].copy()
+ return stime_entities
+
+
+def normalize_row(raw_time, model_name: str, prompt_path: Path) -> str | None:
+ """Normalize a single raw simulation time string using the LLM.
+
+ Parameters
+ ----------
+ raw_time: The raw simulation time string to normalize.
+ model_name: The name of the LLM model to use for normalization.
+ prompt_path: Path to the prompt file for the LLM.
+
+ Returns
+ -------
+ str | None: The normalized simulation time in nanoseconds,
+ or None if normalization fails.
+ """
+ result = normalize_simulation_time(
+ raw_simulation_time=str(raw_time),
+ model_name=model_name,
+ prompt_file_path=prompt_path,
+ )
+ if result and result[0]:
+ return result[0]
+
+
+def normalize_dataframe_times(
+ stime_entities: pd.DataFrame,
+ prompt_path: Path,
+ model_name: str,
+ column: str = "entity",
+) -> pd.DataFrame:
+ """Apply LLM normalization to the specified column of the DataFrame.
+
+ Returns
+ -------
+ pd.DataFrame: The input DataFrame with an additional column 'normalized_time'
+ containing the normalized values.
+ """
+ stime_entities["normalized_time"] = stime_entities[column].apply(
+ lambda x: normalize_row(
+ raw_time=x, model_name=model_name, prompt_path=prompt_path
+ )
+ )
+ return stime_entities
+
+
+def get_llm_normalization_results(stime: pd.DataFrame) -> list[dict]:
+ """Extract LLM normalization results from the DataFrame.
+
+ Parameters
+ ----------
+ stime: DataFrame containing the original entities and their normalized times.
+
+ Returns
+ -------
+ list[dict]: A list of dictionaries with keys 'STIME', 'LLM_value',
+ and 'LLM_unit'.
+ """
+ results = []
+ for _, row in stime.iterrows():
+ llm_outputs = []
+ outputs = json.loads(row["normalized_time"])["output"]
+ for output in outputs:
+ if output.get("value") is not None and output.get("unit") is not None:
+ unit = output["unit"].strip().lower()
+ if unit in UNITS_TO_NS:
+ llm_outputs.append((float(output["value"]), unit))
+
+ if not llm_outputs:
+ llm_outputs = [(None, None)]
+
+ for value, unit in llm_outputs:
+ results.append(
+ {
+ "STIME": str(row["entity"]),
+ "LLM_value": value if value is not None else "None",
+ "LLM_unit": unit if unit is not None else "None",
+ }
+ )
+ return results
+
+
+def save_results_to_tsv(results: list, output_file: Path):
+ """Save the LLM normalization results to a TSV file.
+
+ Parameters
+ ----------
+ results: A list of dictionaries containing the normalization results.
+ output_file: Path to the output TSV file.
+ """
+ results_df = pd.DataFrame(results)
+ results_df_sorted = results_df.sort_values(by="STIME")
+ results_df_sorted.to_csv(output_file, sep="\t", index=False)
+ print(f"\nFile TSV saved : {output_file} ({len(results_df_sorted)} lines)")
+
+
+@click.command()
+@click.option(
+ "--entities-path",
+ type=click.Path(exists=True, dir_okay=False, path_type=Path),
+ help="Path to the TSV file containing the entities.",
+)
+@click.option(
+ "--normalization-results-path",
+ type=click.Path(dir_okay=False, path_type=Path),
+ help="Path to the output TSV file for normalized results.",
+)
+@click.option(
+ "--prompt-path",
+ type=click.Path(file_okay=True, path_type=Path),
+ help="Path to the llm prompt file",
+)
+@click.option(
+ "--model-name",
+ type=str,
+ help="Name of the LLM model to use for normalization.",
+)
+def main(
+ entities_path: Path,
+ normalization_results_path: Path,
+ prompt_path: Path,
+ model_name: str,
+):
+ """Execute the normalization process."""
+ stime_entities = get_stime_entities(entities_path)
+ stime_entities = normalize_dataframe_times(stime_entities, prompt_path, model_name)
+ results = get_llm_normalization_results(stime_entities)
+ save_results_to_tsv(results, normalization_results_path)
+
+
+if __name__ == "__main__":
+ main()