diff --git a/reward_model_training/README.md b/reward_model_training/README.md
new file mode 100644
index 0000000..163e60c
--- /dev/null
+++ b/reward_model_training/README.md
@@ -0,0 +1,30 @@
+# 📁 File Structure:
+
+reward_model_training/
+├── README.md # This guide
+├── main.ipynb # 🎯 MAIN NOTEBOOK - Run this!
+├── rm_util.py # utility class for training, loading, and calling the reward model
+├── reward_data.jsonl # training data used for training the reward model
+├── requirements.txt # Python dependencies
+└── reward_model # The saved trained reward model directory.
+
+
+# Train a reward model:
+
+We'll train the base model microsoft/deberta-v3-base into a reward model with training data from reward_data.jsonl. The trained reward model is saved in 'reward_model' dir.
+
+# Evaluate summaries using the reward model:
+
+We'll load the trained reward model and use it to grade the sample summaries. We'll also evaluate with ROUGE and BERTScore.
+
+# 🎯 Run the main.ipynb to conduct the above tasks:
+
+```bash
+jupyter notebook main.ipynb
+```
+
+# 🔧 Utility Files
+
+rm_util.py - RewardModelUtil class that has the functionality of training a reward model, loading the reward model, and scoring for summarizations provided.
+
+If the 'reward_model' direcotry is present, the training function will not be called; if this directory does not exist, the training function will be executed.
\ No newline at end of file
diff --git a/reward_model_training/main.ipynb b/reward_model_training/main.ipynb
new file mode 100644
index 0000000..4f8b147
--- /dev/null
+++ b/reward_model_training/main.ipynb
@@ -0,0 +1,274 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "aec83135",
+ "metadata": {},
+ "source": [
+ "# Train the reward model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "a90e6b47",
+ "metadata": {
+ "vscode": {
+ "languageId": "plaintext"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "🟡 Using MPS (Apple Silicon GPU)\n",
+ "Base model: microsoft/deberta-v3-base\n",
+ "Training microsoft/deberta-v3-base with reward_data.jsonl\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at microsoft/deberta-v3-base and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\n",
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+ "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/transformers/convert_slow_tokenizer.py:564: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
+ " warnings.warn(\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "9bd7eb17d28641d58a77a56da173d71f",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Filtering train >1024 tokens: 0%| | 0/10 [00:00\n",
+ " \n",
+ " \n",
+ " [6/6 00:18, Epoch 3/3]\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Step | \n",
+ " Training Loss | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/torch/utils/data/dataloader.py:684: UserWarning: 'pin_memory' argument is set as true but not supported on MPS now, then device pinned memory won't be used.\n",
+ " warnings.warn(warn_msg)\n",
+ "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/torch/utils/data/dataloader.py:684: UserWarning: 'pin_memory' argument is set as true but not supported on MPS now, then device pinned memory won't be used.\n",
+ " warnings.warn(warn_msg)\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Loading trained reward model...\n",
+ "✅ trained reward model loaded successfully\n"
+ ]
+ }
+ ],
+ "source": [
+ "import json\n",
+ "from rm_util import RewardModelUtil\n",
+ "from evaluate import load\n",
+ "\n",
+ "rm = RewardModelUtil()\n",
+ "rm.train_model('reward_data.jsonl')\n",
+ "rm.load_model()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "633707f2",
+ "metadata": {},
+ "source": [
+ "# Sample summary data to be evaluated with ROUGE, BERTScore, and the trained reward model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "c4c24458",
+ "metadata": {
+ "vscode": {
+ "languageId": "plaintext"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "generated_summaries: ['The architectures of LNU-Net and IBU-Net have a down-sampling path for feature extraction and an up-sampling path for precise localization. We use the original U-Net as the basic segmentation approach and compared it with our proposed architectures.', 'Integrating symbolic constraints into deep learning models could make them more robust but this is a time-consuming and challenging task. In this paper we propose AgenticDomiKnowS (ADS) to eliminate this dependency. ADS translates free-form task descriptions into a complete DomiKnowS program and supports optional human-in-the-loop intervention. We show how ADS enables experienced DomiKnowS users and non-users to rapidly construct neuro-symbolic programs, reducing development time from hours to 10-15 minutes.', 'LLMs need continual learning because the knowledge of LLMs quickly becomes outdated as data evolve. Memory-augmented approaches address this by equipping LLMs with a memory bank, that is an external memory module which stores information for future use. However, the memory bank constantly grows in the real-world scenario. In this paper, we propose MBC, a model that compresses the memory bank through a codebook optimization strategy during online adaptation learning.']\n",
+ "reference_summaries: ['This papar introduces two deep learning approaches called LNU-Net and IBU-Net for Automated Segmentation of Left Ventricle in Cine Cardiac MRI', 'This paper proposes AgenticDomiKnowS (ADS) which eliminates the dependency of integrating symbolic constraints into deep learning models, and shows how ADS enables experienced DomiKnowS users and non-users to rapidly construct neuro-symbolic programs.', 'Memory bank is an external memory module which stores information for LLM to get new knowledge. This paper proposes MBC model that addresses the problem of the memory bank constantly growing, by compressing the memory bank through a codebook optimization strategy during online adaptation learning, and employing Key-Value Low-Rank Adaptation in the attention layers of the LLM.']\n"
+ ]
+ }
+ ],
+ "source": [
+ "sample_summaries = [\n",
+ " {\n",
+ " 'original_content': 'Left ventricle (LV) segmentation is critical for clinical quantification and diagnosis of cardiac images. In this work, we propose two novel deep learning architectures called LNU-Net and IBU-Net for left ventricle segmentation from short-axis cine MRI images. LNU-Net is derived from layer normalization (LN) U-Net architecture, while IBU-Net is derived from the instance-batch normalized (IB) U-Net for medical image segmentation. The architectures of LNU-Net and IBU-Net have a down-sampling path for feature extraction and an up-sampling path for precise localization. We use the original U-Net as the basic segmentation approach and compared it with our proposed architectures. Both LNU-Net and IBU-Net have left ventricle segmentation methods: LNU-Net applies layer normalization in each convolutional block, while IBU-Net incorporates instance and batch normalization together in the first convolutional block and passes its result to the next layer. Our method incorporates affine transformations and elastic deformations for image data processing. Our dataset that contains 805 MRI images regarding the left ventricle from 45 patients is used for evaluation. We experimentally evaluate the results of the proposed approaches outperforming the dice coefficient and the average perpendicular distance than other state-of-the-art approaches.',\n",
+ " 'generated_summary': 'The architectures of LNU-Net and IBU-Net have a down-sampling path for feature extraction and an up-sampling path for precise localization. We use the original U-Net as the basic segmentation approach and compared it with our proposed architectures.',\n",
+ " 'reference_summary': 'This papar introduces two deep learning approaches called LNU-Net and IBU-Net for Automated Segmentation of Left Ventricle in Cine Cardiac MRI'\n",
+ " },\n",
+ " {\n",
+ " 'original_content': 'Integrating symbolic constraints into deep learning models could make them more robust, interpretable, and data-efficient. Still, it remains a time-consuming and challenging task. Existing frameworks like DomiKnowS help this integration by providing a high-level declarative programming interface, but they still assume the user is proficient with the library\\'s specific syntax. We propose AgenticDomiKnowS (ADS) to eliminate this dependency. ADS translates free-form task descriptions into a complete DomiKnowS program using an agentic workflow that creates and tests each DomiKnowS component separately. The workflow supports optional human-in-the-loop intervention, enabling users familiar with DomiKnowS to refine intermediate outputs. We show how ADS enables experienced DomiKnowS users and non-users to rapidly construct neuro-symbolic programs, reducing development time from hours to 10-15 minutes.',\n",
+ " 'generated_summary': 'Integrating symbolic constraints into deep learning models could make them more robust but this is a time-consuming and challenging task. In this paper we propose AgenticDomiKnowS (ADS) to eliminate this dependency. ADS translates free-form task descriptions into a complete DomiKnowS program and supports optional human-in-the-loop intervention. We show how ADS enables experienced DomiKnowS users and non-users to rapidly construct neuro-symbolic programs, reducing development time from hours to 10-15 minutes.',\n",
+ " 'reference_summary': 'This paper proposes AgenticDomiKnowS (ADS) which eliminates the dependency of integrating symbolic constraints into deep learning models, and shows how ADS enables experienced DomiKnowS users and non-users to rapidly construct neuro-symbolic programs.'\n",
+ " },\n",
+ " {\n",
+ " 'original_content': 'Large Language Models (LLMs) have become a mainstay for many everyday applications. However, as data evolve their knowledge quickly becomes outdated. Continual learning aims to update LLMs with new information without erasing previously acquired knowledge. Although methods such as full fine-tuning can incorporate new data, they are computationally expensive and prone to catastrophic forgetting, where prior knowledge is overwritten. Memory-augmented approaches address this by equipping LLMs with a memory bank, that is an external memory module which stores information for future use. However, these methods face a critical limitation, in particular, the memory bank constantly grows in the real-world scenario when large-scale data streams arrive. In this paper, we propose MBC, a model that compresses the memory bank through a codebook optimization strategy during online adaptation learning. To ensure stable learning, we also introduce an online resetting mechanism that prevents codebook collapse. In addition, we employ Key-Value Low-Rank Adaptation in the attention layers of the LLM, enabling efficient utilization of the compressed memory representations. Experiments with benchmark question-answering datasets demonstrate that MBC reduces the memory bank size to 0.3% when compared against the most competitive baseline, while maintaining high retention accuracy during online adaptation learning. Our code is publicly available at https://github.com/Thomkat/MBC.',\n",
+ " 'generated_summary': 'LLMs need continual learning because the knowledge of LLMs quickly becomes outdated as data evolve. Memory-augmented approaches address this by equipping LLMs with a memory bank, that is an external memory module which stores information for future use. However, the memory bank constantly grows in the real-world scenario. In this paper, we propose MBC, a model that compresses the memory bank through a codebook optimization strategy during online adaptation learning.',\n",
+ " 'reference_summary': 'Memory bank is an external memory module which stores information for LLM to get new knowledge. This paper proposes MBC model that addresses the problem of the memory bank constantly growing, by compressing the memory bank through a codebook optimization strategy during online adaptation learning, and employing Key-Value Low-Rank Adaptation in the attention layers of the LLM.'\n",
+ " }\n",
+ "]\n",
+ "\n",
+ "generated_summaries = [entry['generated_summary'] for entry in sample_summaries]\n",
+ "reference_summaries = [entry['reference_summary'] for entry in sample_summaries]\n",
+ "\n",
+ "print(f'generated_summaries: {generated_summaries}')\n",
+ "print(f'reference_summaries: {reference_summaries}')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a37e1998",
+ "metadata": {},
+ "source": [
+ "# ROUGE and BERTScore:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "da09d1fc",
+ "metadata": {
+ "vscode": {
+ "languageId": "plaintext"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']\n",
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ROUGE: {'rouge1': np.float64(0.4496096801440313), 'rouge2': np.float64(0.31125154231618923), 'rougeL': np.float64(0.3524266486098547), 'rougeLsum': np.float64(0.3524266486098547)}\n",
+ "BERTScore: {'precision': [0.8630497455596924, 0.8990731239318848, 0.9037832021713257], 'recall': [0.8537697792053223, 0.9558082818984985, 0.910071611404419], 'f1': [0.8583846688270569, 0.9265730381011963, 0.9069164991378784], 'hashcode': 'roberta-large_L17_no-idf_version=0.3.12(hug_trans=4.57.1)'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "rouge = load(\"rouge\")\n",
+ "bertscore = load(\"bertscore\")\n",
+ "\n",
+ "results_rouge = rouge.compute(predictions=generated_summaries, references=reference_summaries)\n",
+ "results_bertscore = bertscore.compute(predictions=generated_summaries, references=reference_summaries, lang=\"en\")\n",
+ "\n",
+ "print(\"ROUGE:\", results_rouge)\n",
+ "print(\"BERTScore:\", results_bertscore)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b4287853",
+ "metadata": {},
+ "source": [
+ "# Reward Model Scores on the generated summaries and reference summaries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "bbbe444c",
+ "metadata": {
+ "vscode": {
+ "languageId": "plaintext"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Reward Model Scores: generated summary: -0.16497120261192322; reference summary: -0.1577276885509491\n",
+ "Reward Model Scores: generated summary: -0.15327253937721252; reference summary: -0.12794075906276703\n",
+ "Reward Model Scores: generated summary: -0.17355310916900635; reference summary: -0.1665564775466919\n"
+ ]
+ }
+ ],
+ "source": [
+ "for entry in sample_summaries:\n",
+ " score1, score2 = rm.score_summaries(entry)\n",
+ " print(f'Reward Model Scores: generated summary: {score1}; reference summary: {score2}')"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.13.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/reward_model_training/requirements.txt b/reward_model_training/requirements.txt
new file mode 100644
index 0000000..230ecbf
--- /dev/null
+++ b/reward_model_training/requirements.txt
@@ -0,0 +1,4 @@
+evaluate
+nltk
+rouge_score
+bert_score
\ No newline at end of file
diff --git a/reward_model_training/reward_data.jsonl b/reward_model_training/reward_data.jsonl
new file mode 100644
index 0000000..b9827b5
--- /dev/null
+++ b/reward_model_training/reward_data.jsonl
@@ -0,0 +1,10 @@
+{"chosen": "A training-free method uses spectral analysis of attention patterns to detect valid mathematical reasoning in large language models. The approach achieves high accuracy (85-95%) across multiple architectures without fine-tuning, using only a single threshold on a spectral metric. It also detects logical coherence rather than compiler acceptance.", "rejected": "The authors present a novel method, without any training data or fine-tuning, that can accurately detect valid mathematical reasoning in large language models using spectral analysis of attention patterns. They treat attention matrices as graphs over tokens and extract four interpretable metrics: Fiedler value, high-frequency energy ratio, graph signal smoothness, and spectral entropy. These metrics show statistically significant differences between valid and invalid mathematical proofs. The method achieves high classification accuracy (85-95%) on 7 transformer models from different architectural families, demonstrating a Cohen's effect size of up to 3.30 and p-value less than 10^-116. The authors also discover that the method detects logical coherence rather than compiler acceptance, identifying valid proofs that formal verifiers reject due to technical failures. Additionally, they find that attention mechanism design affects which spectral features capture reasoning validity, and this principled framework for reasoning verification has potential applications in hallucination detection and AI safety monitoring."}
+{"chosen": "A new framework, FedHypeVAE, generates heterogeneous client data while preserving privacy and personalization. It uses a conditional VAE with client-aware decoders and class-conditional priors generated by a shared hypernetwork optimized for differential privacy. The framework improves stability, distribution alignment, and multi-domain coverage, enabling principled privacy-preserving data synthesis in federated settings.", "rejected": "The authors propose FedHypeVAE, a framework that enables decentralized clients to share embedding-level data while maintaining data privacy and personalization. The approach uses hypernetworks to generate class-conditional priors and client-aware decoders, replacing traditional global decoders and fixed latent priors. This bi-level design is optimized under differential privacy to ensure only noisy gradients are shared across clients. To improve stability and distributional coherence, the authors use a local maximum mean discrepancy (MMD) alignment between real and synthetic embeddings, as well as a Lipschitz regularizer on hypernetwork outputs. The trained model can then synthesize data in new domains using a neutral meta-code or mixtures of meta-codes for controlled multi-domain coverage. Overall, FedHypeVAE provides a principled foundation for privacy-preserving data synthesis in federated settings."}
+{"chosen": "he paper develops a framework for contrasting warped brane inflation models with cosmic microwave background observations, considering factors like moduli stabilization and ultraviolet bulk physics. It finds that without bulk effects, some parameters yield consistent predictions, but when included, predictions are generally spoiled, requiring fine-tuning to maintain consistency.", "rejected": "The authors propose a comprehensive framework for comparing observations with a warped brane inflation model. They provide an example of the inflaton potential governing the motion of a D3 brane in the entire warped deformed conifold, allowing them to identify corresponding scales of the cosmic microwave background. The effects of bulk fluxes or localized sources are parametrized using gauge/string duality. Simulation results show that without considering bulk effects, there can be multiple sets of parameters that produce observationally consistent predictions. However, when bulk perturbations are included, these predictions are typically spoiled and require fine-tuning to remain consistent with observations."}
+{"chosen": "The Blazar 3C 454.3 was observed in a high energy state by Fermi's Gamma-ray Space Telescope in July 2008, prompting an international team to monitor its multi-wavelength emission. The results showed excellent correlations between IR, optical, UV, and gamma-ray light curves, with the X-rays not strongly linked to these wavelengths.", "rejected": "The Fermi Gamma-ray Space Telescope detected an unusually high level of activity in the blazar 3C 454.3 in July 2008. In response, researchers conducted a multi-wavelength monitoring campaign using data from various telescopes (SMARTS, Swift, and Fermi) to study this phenomenon. The results show that the infrared, optical, ultraviolet, and gamma-ray light curves are closely correlated with each other, with a delay of less than one day. The fluctuations in infrared radiation are similar in magnitude to those seen in gamma-rays, while they are larger at longer wavelengths (optical, UV) and smaller in X-ray data. The variability patterns can be explained by the \"external Compton model\", which suggests that high-energy electrons produce synchrotron emission and scatter light from an accretion disk or photons at shorter wavelengths to create gamma-rays, while cooler electrons produce X-rays through photon scattering."}
+{"chosen": "The study explores \"general sets of events\" (GSEs), which are sets of states with associated probabilities. GSEs can represent various types of logics, including quantum logic and Boolean algebra. The paper focuses on characterizing GSEs as posets and lattices, particularly those that are orthoposets and their connections to known logics.", "rejected": "The study examines \"general sets of events\" (GSEs), which are sets of probabilities related to physical systems. A GSE consists of sets (S) and probability functions (p(s)) where p(s) represents the likelihood of an event occurring when the system is in state s. The authors show that various well-studied algebraic structures, including those found in quantum logic and Boolean algebras, can be represented within this framework.\n\nThe paper focuses on different classes of GSEs, particularly those called \"orthoposets\" which are studied along with their connections to known logics. It also explores when GSEs can be considered lattices (a specific type of mathematical structure), and characterizes these sets using states rather than probability functions.\n\nIn summary, the research aims to understand the properties and relationships between different classes of general sets of events, which are used as a foundation for various types of logic."}
+{"chosen": "The paper introduces an efficient classical algorithm that calculates stabilizer R\u00e9nyi entropies and nullity for many-body wavefunctions of qubits, achieving exponential speedup over direct approaches. It also develops a Monte-Carlo estimator with variance reduction scheme to quantify \"quantum magic\" and applies it to various quantum states and circuits.", "rejected": "The study focuses on developing efficient classical algorithms for computing stabilizer R\u00e9nyi entropies and nullity for many-body wavefunctions. The proposed approach combines a fast Walsh-Hadamard transform with an exact partition of Pauli operators, resulting in an exponential speedup over direct methods (reducing the computational cost from O(2^N) to O(N)). The method is then extended to develop a Monte-Carlo estimator for stabilizer R\u00e9nyi entropies, along with a Clifford-based variance-reduction scheme. The accuracy and efficiency of the approach are demonstrated through simulations using random magic states and doped Clifford circuits. The method applies universally to arbitrary quantum states and provides a quantitative measure of \"quantum magic\" resources encoded in highly entangled states or generated by long-time nonequilibrium dynamics."}
+{"chosen": "The paper derives a formula for correlation energy in a two-dimensional Fermi gas with certain potential types, including Coulomb potential. Using bosonization and patch-based analysis, it establishes an upper bound for the correlation energy. The proof requires refining low-energy excitation analysis due to fewer but larger contributions compared to 3D cases.", "rejected": "The authors have developed a formula for calculating the correlation energy of a two-dimensional gas consisting of fermions (particles with half-integer spin) at very low temperatures, where interactions between particles can be approximated as mean-field-like. They show that this formula works for any potential energy function V(k), where k represents the wavenumber, as long as the absolute value of the Fourier transform of V satisfies specific mathematical conditions. One example of such a potential is the Coulomb potential, which is proportional to k^(-2). The proof involves a mathematical technique called bosonization and requires a detailed analysis of low-energy particle excitations, which are less common but more significant in two-dimensional systems compared to three-dimensional ones."}
+{"chosen": "The study explores a massive scalar field theory that breaks Lorentz symmetry through a background tensor, then applies Thermo Field Dynamics to examine space-time compactification effects. The result shows corrections to energy-momentum tensor and Feynman propagator, influencing Stefan-Boltzmann law and Casimir effect behaviors.", "rejected": "The study investigates a new scalar field theory that breaks Lorentz symmetry (a fundamental concept in physics) and incorporates effects from compactified space-time (imagine shrinking dimensions). The approach uses Thermo Field Dynamics (TFD), which allows researchers to treat both thermal and finite-size phenomena simultaneously. By analyzing the modified energy-momentum tensor and Feynman propagator, the study finds that Lorentz-violating backgrounds alter the behavior of quantum fields in two important areas: the Stefan-Boltzmann law (which describes radiation absorption/emission) and the Casimir effect (a phenomenon where forces arise between two parallel plates). The research highlights how temperature, spatial constraints, and Lorentz symmetry all interact to shape the behavior of these quantum fields."}
+{"chosen": "The study proposes Rogue Variable Theory (RVT), which formalizes cognitive configurations that exist before events are fully understood. RVT creates a framework for processing these states using graph theory and information-theory metrics, enabling analysis of human cognition without requiring physical quantum processes.", "rejected": "The paper discusses how many important cognitive processes occur before explicit events take place, such as decision-making or labeling emotions. These pre-event states are characterized by uncertainty, tension, and competing interpretations. The researchers propose a theory called Rogue Variable Theory (RVT) to formalize these states.\n\nThey describe RVT as \"Rogue Variables,\" which are structured configurations that affect outcomes even if they don't fit into the current system's representation. To implement this theory, they use a quantum-consistent information-theoretic framework based on a graph structure known as a Mirrored Personal Graph (MPG). They also introduce a Quantum MPG State (QMS) and an \"rogue operator\" that helps identify rogue factors and candidate Rogue Variable segments.\n\nFurthermore, the researchers present a layer called the Rosetta Stone Layer (RSL), which allows for comparing and aggregating user-specific latent factor coordinates in a shared reference space without requiring explicit node alignment. This means that their framework can be implemented on classical systems and doesn't rely on physical quantum processes, but rather interprets \"collapse\" as information decoherence under interaction or human clarification."}
+{"chosen": "This paper develops a framework for analyzing dynamic adaptive experiments where data collection and treatment assignment evolve over time. It introduces a limit representation using Gaussian diffusions to facilitate analysis of optimal rules, estimation, and anytime-valid inference for multi-treatment settings.", "rejected": "The article proposes a new framework for analyzing adaptive experiments where data collection and treatment assignment change dynamically over time based on new information. The key challenge is that the sequence of policy rules can be complex and difficult to analyze. Instead, the authors focus on the empirical allocation process, which shows the proportion of observations assigned to each treatment over time. They show that this empirical process can be approximated by a Gaussian diffusion process with unknown drifts. This limit representation simplifies the analysis of optimal decision rules and enables the derivation of optimal estimators, in-sample regret analysis for adaptive experiments, and construction of processes for anytime-valid inference. The framework also introduces the first definition of valid inference for multi-treatment settings at any time or experiment."}
diff --git a/reward_model_training/rm_util.py b/reward_model_training/rm_util.py
new file mode 100644
index 0000000..7f4a175
--- /dev/null
+++ b/reward_model_training/rm_util.py
@@ -0,0 +1,140 @@
+import os
+import torch
+from trl import RewardTrainer, RewardConfig
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from datasets import load_dataset
+
+class RewardModelUtil:
+ """Util class to train a reward model on summarization evaluation"""
+
+ def __init__(self, model_path: str = "microsoft/deberta-v3-base"):
+ """
+ Args:
+ model_path: Path of base model to be trained
+ """
+ self.base_model_path = model_path
+ self.reward_model_path = "./reward_model"
+ self.tokenizer = None
+ self.model = None
+ if torch.cuda.is_available():
+ self.device = torch.device("cuda")
+ print("✅ Using CUDA (GPU)")
+ elif torch.backends.mps.is_available():
+ self.device = torch.device("mps")
+ print("🟡 Using MPS (Apple Silicon GPU)")
+ else:
+ self.device = torch.device("cpu")
+ print("🔴 Using CPU")
+ print(f"Base model: {model_path}")
+
+ def train_model(self, data_file):
+ if os.path.exists(self.reward_model_path):
+ return
+
+ print(f"Training {self.base_model_path} with {data_file}")
+
+ base_model = AutoModelForSequenceClassification.from_pretrained(self.base_model_path, num_labels=1)
+ # Monkey-Patch the Forward Pass
+ original_forward = base_model.forward
+ def patched_forward(*args, **kwargs):
+ kwargs.pop("use_cache", None) # Remove use_cache if it exists
+ return original_forward(*args, **kwargs)
+
+ base_model.forward = patched_forward
+ #base_model.config.use_cache = False # Keep this as a backup
+
+ tokenizer = AutoTokenizer.from_pretrained(self.base_model_path)
+ if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
+ tokenizer.pad_token_id = tokenizer.eos_token_id
+
+ dataset = load_dataset("json", data_files=data_file, split="train")
+
+ def preprocess(example):
+ return tokenizer(example["chosen"], example["rejected"], truncation=True, padding="max_length")
+
+ dataset = dataset.map(preprocess, batched=True)
+
+ training_args = RewardConfig(
+ output_dir=self.reward_model_path,
+ per_device_train_batch_size=8,
+ num_train_epochs=3,
+ save_strategy="epoch",
+ logging_steps=10,
+ fp16=False,
+ bf16=True, # Use bfloat16 for stability
+ disable_dropout=True, # Standard RewardConfig often defaults disable_dropout to True
+ gradient_checkpointing=True,
+ gradient_checkpointing_kwargs={"use_reentrant": False}, # CRITICAL FOR 2026
+ remove_unused_columns=False
+ )
+
+ trainer = RewardTrainer(
+ model=base_model,
+ args=training_args,
+ train_dataset=dataset,
+ )
+
+ trainer.train()
+ trainer.save_model(self.reward_model_path)
+
+
+ def load_model(self):
+ if os.path.exists(self.reward_model_path):
+ print(f"Loading trained reward model...")
+ self.model = AutoModelForSequenceClassification.from_pretrained(
+ self.reward_model_path
+ )
+ self.tokenizer = AutoTokenizer.from_pretrained(self.reward_model_path)
+ if self.tokenizer.pad_token is None:
+ self.tokenizer.pad_token = self.tokenizer.eos_token
+ print(f"✅ trained reward model loaded successfully")
+
+
+ def score_summaries(self, entry):
+ self.model.to(self.device)
+ self.model.eval()
+
+ prompt = f'''
+Summarize the following research paper excerpt:
+
+{entry['original_content']}
+ '''
+
+ inputs = self.tokenizer(
+ prompt,
+ entry['generated_summary'],
+ truncation=True,
+ padding="max_length",
+ max_length=512,
+ return_tensors="pt"
+ ).to(self.device) # Ensure you move to GPU
+
+ with torch.no_grad():
+ outputs = self.model(**inputs)
+
+ # Extract the scalar score
+ # In a SequenceClassification model with 1 label, the score is in the logits
+ if self.device != torch.device("cuda"):
+ score1 = outputs.logits.detach().cpu().item()
+ else:
+ score1 = outputs.logits.item()
+
+ inputs = self.tokenizer(
+ prompt,
+ entry['reference_summary'],
+ truncation=True,
+ padding="max_length",
+ max_length=512,
+ return_tensors="pt"
+ ).to(self.device)
+
+ with torch.no_grad():
+ outputs = self.model(**inputs)
+
+ if self.device != torch.device("cuda"):
+ score2 = outputs.logits.detach().cpu().item()
+ else:
+ score2 = outputs.logits.item()
+
+ return score1, score2
\ No newline at end of file