Skip to content

Fine grained breakdown of novelty/uniqueness & symprec sensitivity #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smglsn12 wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Fine grained breakdown of novelty/uniqueness & symprec sensitivity #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9bc7588
feat: fine grained breakdown for novelty and uniqueness, symprec swee…
smglsn12 Apr 17, 2026
d6230f5
chore: run ruff checks
smglsn12 Apr 17, 2026
f4e9861
chore: run ruff checks try 2
smglsn12 Apr 17, 2026
a7d6a22
Chore: increase plot fontsize for publication
smglsn12 Apr 18, 2026
a30ffdc
feat: update novelty metric so it checks for novel prototypes in LeMa…
smglsn12 Apr 19, 2026
174a761
chore: fix linter errors
smglsn12 Apr 19, 2026
e539961
feat: add frameowrk novelty and update figure creation scripts
smglsn12 Apr 27, 2026
29df6c9
chore: run ruff checks
smglsn12 Apr 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions data/lematbulk_anon_sg_index.json
View file
Open in desktop

Large diffs are not rendered by default.

1,692 changes: 846 additions & 846 deletions notebooks/example_distribution_metrics.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

388 changes: 194 additions & 194 deletions notebooks/example_stability_benchmark.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,197 +1,197 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "495e6499-b0d4-429f-8ac3-febaaa06abc5",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from datasets import load_dataset\n",
"from pymatgen.core import Composition, Element, Structure\n",
"from pymatgen.util.testing import PymatgenTest\n",
"\n",
"from lemat_genbench.benchmarks.stability_benchmark import StabilityBenchmark\n",
"from lemat_genbench.benchmarks.validity_benchmark import ValidityBenchmark\n",
"from lemat_genbench.metrics.stability_metrics import StabilityMetric\n",
"from lemat_genbench.preprocess.reference_energies import (\n",
" get_energy_above_hull,\n",
" get_formation_energy_from_composition_energy,\n",
")\n",
"from lemat_genbench.preprocess.stability_preprocess import (\n",
" EnergyAboveHull,\n",
" OrbFormationEnergy,\n",
" StabilityPreprocessor,\n",
")\n",
"\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "b344162e-9448-40d3-b748-f52f52ff798e",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\samue\\AppData\\Local\\Temp\\ipykernel_6556\\2264363178.py:1: FutureWarning: PymatgenTest is deprecated, and will be removed on 2026-01-01\n",
"Use MatSciTest in pymatgen.util.testing instead.\n",
" test = PymatgenTest()\n"
]
}
],
"source": [
"test = PymatgenTest()\n",
"\n",
"filename = \"CsBr.cif\"\n",
"structure = Structure.from_file(filename)\n",
"structure = structure.remove_oxidation_states()\n",
"\n",
"filename2 = \"CsPbBr3.cif\"\n",
"structure2 = Structure.from_file(filename2)\n",
"structure2 = structure2.remove_oxidation_states()\n",
"\n",
"structures = [\n",
" structure,\n",
" structure2,\n",
" test.get_structure(\"Si\"),\n",
" test.get_structure(\"LiFePO4\"),\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 190,
"id": "ed5a490e-bfb0-443e-a730-c00dcdd85140",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cpu\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\samue\\lematerial-forgebench\\.venv\\Lib\\site-packages\\orb_models\\utils.py:30: UserWarning: Setting global torch default dtype to torch.float32.\n",
" warnings.warn(f\"Setting global torch default dtype to {torch_dtype}.\")\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"energy_above_hull : 0.0\n",
"energy_above_hull relaxed : 0.0\n",
"energy_above_hull : 0.0322531480375976\n",
"energy_above_hull relaxed : 0.028652646023437534\n",
"energy_above_hull : 0.010958068559570044\n",
"energy_above_hull relaxed : 0.009661548326415748\n",
"energy_above_hull : 0.24325226729073446\n",
"energy_above_hull relaxed : 0.24249804714564505\n"
]
}
],
"source": [
"stability_preprocessor = StabilityPreprocessor()\n",
"preprocessor_result = stability_preprocessor(structures)"
]
},
{
"cell_type": "code",
"execution_count": 191,
"id": "34e1ae49-7e44-4ea5-a826-073f2b23d01d",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cs4 Br4 Formation Energy : -7.149936223230794\n",
"Cs4 Pb4 Br12 Formation Energy : -8.492938243082682\n",
"Si2 Formation Energy : 0.02191613711914009\n",
"Li4 Fe4 P4 O16 Formation Energy : -12.061907388726752\n",
"Cs4 Br4 Relaxation Stability RMSE : 0.030938701621605395\n",
"Cs4 Pb4 Br12 Relaxation Stability RMSE : 0.1821416738236651\n",
"Si2 Relaxation Stability RMSE : 0.022634037275015083\n",
"Li4 Fe4 P4 O16 Relaxation Stability RMSE : 0.056377222390659246\n",
"{'stable_ratio': np.float64(0.25), 'metastable_ratio': np.float64(0.75), 'mean_e_above_hull': np.float64(0.07161587097197553), 'mean_formation_energy': np.float64(-6.920716429480272), 'mean_relaxation_RMSE': np.float64(0.0730229087777362)}\n"
]
}
],
"source": [
"benchmark = StabilityBenchmark()\n",
"benchmark_result = benchmark.evaluate(preprocessor_result.processed_structures)"
]
},
{
"cell_type": "code",
"execution_count": 192,
"id": "aa87ce40-480e-40f2-9a27-7701fa89cbc3",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"{'stable_ratio': np.float64(0.25),\n",
" 'metastable_ratio': np.float64(0.75),\n",
" 'mean_e_above_hull': np.float64(0.07161587097197553),\n",
" 'mean_formation_energy': np.float64(-6.920716429480272),\n",
" 'mean_relaxation_RMSE': np.float64(0.0730229087777362)}"
]
},
"execution_count": 192,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"benchmark_result.final_scores"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f10bfb01-e55d-465b-bb91-3274255f8bad",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "495e6499-b0d4-429f-8ac3-febaaa06abc5",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from datasets import load_dataset\n",
"from lemat_genbench.benchmarks.stability_benchmark import StabilityBenchmark\n",
"from lemat_genbench.metrics.stability_metrics import StabilityMetric\n",
"from lemat_genbench.preprocess.stability_preprocess import (\n",
" EnergyAboveHull,\n",
" OrbFormationEnergy,\n",
" StabilityPreprocessor,\n",
")\n",
"from pymatgen.core import Composition, Element, Structure\n",
"from pymatgen.util.testing import PymatgenTest\n",
"\n",
"from lemat_genbench.benchmarks.validity_benchmark import ValidityBenchmark\n",
"from lemat_genbench.preprocess.reference_energies import (\n",
" get_energy_above_hull,\n",
" get_formation_energy_from_composition_energy,\n",
")\n",
"\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "b344162e-9448-40d3-b748-f52f52ff798e",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\samue\\AppData\\Local\\Temp\\ipykernel_6556\\2264363178.py:1: FutureWarning: PymatgenTest is deprecated, and will be removed on 2026-01-01\n",
"Use MatSciTest in pymatgen.util.testing instead.\n",
" test = PymatgenTest()\n"
]
}
],
"source": [
"test = PymatgenTest()\n",
"\n",
"filename = \"CsBr.cif\"\n",
"structure = Structure.from_file(filename)\n",
"structure = structure.remove_oxidation_states()\n",
"\n",
"filename2 = \"CsPbBr3.cif\"\n",
"structure2 = Structure.from_file(filename2)\n",
"structure2 = structure2.remove_oxidation_states()\n",
"\n",
"structures = [\n",
" structure,\n",
" structure2,\n",
" test.get_structure(\"Si\"),\n",
" test.get_structure(\"LiFePO4\"),\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 190,
"id": "ed5a490e-bfb0-443e-a730-c00dcdd85140",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cpu\n"
]
},
"nbformat": 4,
"nbformat_minor": 5
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\samue\\lematerial-forgebench\\.venv\\Lib\\site-packages\\orb_models\\utils.py:30: UserWarning: Setting global torch default dtype to torch.float32.\n",
" warnings.warn(f\"Setting global torch default dtype to {torch_dtype}.\")\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"energy_above_hull : 0.0\n",
"energy_above_hull relaxed : 0.0\n",
"energy_above_hull : 0.0322531480375976\n",
"energy_above_hull relaxed : 0.028652646023437534\n",
"energy_above_hull : 0.010958068559570044\n",
"energy_above_hull relaxed : 0.009661548326415748\n",
"energy_above_hull : 0.24325226729073446\n",
"energy_above_hull relaxed : 0.24249804714564505\n"
]
}
],
"source": [
"stability_preprocessor = StabilityPreprocessor()\n",
"preprocessor_result = stability_preprocessor(structures)"
]
},
{
"cell_type": "code",
"execution_count": 191,
"id": "34e1ae49-7e44-4ea5-a826-073f2b23d01d",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cs4 Br4 Formation Energy : -7.149936223230794\n",
"Cs4 Pb4 Br12 Formation Energy : -8.492938243082682\n",
"Si2 Formation Energy : 0.02191613711914009\n",
"Li4 Fe4 P4 O16 Formation Energy : -12.061907388726752\n",
"Cs4 Br4 Relaxation Stability RMSE : 0.030938701621605395\n",
"Cs4 Pb4 Br12 Relaxation Stability RMSE : 0.1821416738236651\n",
"Si2 Relaxation Stability RMSE : 0.022634037275015083\n",
"Li4 Fe4 P4 O16 Relaxation Stability RMSE : 0.056377222390659246\n",
"{'stable_ratio': np.float64(0.25), 'metastable_ratio': np.float64(0.75), 'mean_e_above_hull': np.float64(0.07161587097197553), 'mean_formation_energy': np.float64(-6.920716429480272), 'mean_relaxation_RMSE': np.float64(0.0730229087777362)}\n"
]
}
],
"source": [
"benchmark = StabilityBenchmark()\n",
"benchmark_result = benchmark.evaluate(preprocessor_result.processed_structures)"
]
},
{
"cell_type": "code",
"execution_count": 192,
"id": "aa87ce40-480e-40f2-9a27-7701fa89cbc3",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"{'stable_ratio': np.float64(0.25),\n",
" 'metastable_ratio': np.float64(0.75),\n",
" 'mean_e_above_hull': np.float64(0.07161587097197553),\n",
" 'mean_formation_energy': np.float64(-6.920716429480272),\n",
" 'mean_relaxation_RMSE': np.float64(0.0730229087777362)}"
]
},
"execution_count": 192,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"benchmark_result.final_scores"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f10bfb01-e55d-465b-bb91-3274255f8bad",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
1 change: 1 addition & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ dependencies = [
"mp-api>=0.45.12",
"aflow>=0.0.11",
"seaborn>=0.13.2",
"pymatviz>=0.17.2",
]

[project.scripts]
Expand Down
Loading
Loading