From aaeb0b472a191b7c2209af5468b005ef8bc1fdee Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Thu, 18 Jun 2026 15:25:58 +0200 Subject: [PATCH 01/21] livinglab installable module --- examples/init_env.py | 4 ---- examples/train_omnisafe.py | 15 +++------------ examples/train_sb3.py | 3 --- pyproject.toml | 30 ++++++++++++++++++++++++++++++ scripts/agent.py | 3 --- scripts/eval.py | 4 +--- scripts/eval_rbc.py | 4 +--- scripts/train.py | 4 +--- 8 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 pyproject.toml diff --git a/examples/init_env.py b/examples/init_env.py index c7d6e27..1a74407 100644 --- a/examples/init_env.py +++ b/examples/init_env.py @@ -1,8 +1,4 @@ -import sys -import os import json -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - import numpy as np from collections import defaultdict diff --git a/examples/train_omnisafe.py b/examples/train_omnisafe.py index e863d33..e0ad284 100644 --- a/examples/train_omnisafe.py +++ b/examples/train_omnisafe.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # Omnisafe from ext import omnisafe @@ -40,7 +37,7 @@ def main(args): 'seed': args.seed, 'train_cfgs': { 'total_steps': args.episodes*(env_cfgs['episode_length']-1), - 'device': 'cuda' if torch.cuda.is_available() else 'cpu' + 'device': 'cuda:0' if torch.cuda.is_available() else 'cpu' }, 'algo_cfgs': { 'steps_per_epoch': env_cfgs['episode_length']-1, @@ -58,18 +55,12 @@ def main(args): # --- LIVINGLAB KEY ARGUMENTS --- 'env_cfgs': { - **env_cfgs, - 'cost_fn': { - 'name': 'electricity_consumption', - 'kwargs': { - 'exponent': 2.0 - } - } + **env_cfgs } } # 3. Define and train the agent - agent = omnisafe.Agent('PPOLag', 'LivingLab-v0', custom_cfgs=custom_cfgs) + agent = omnisafe.Agent('PPO', 'LivingLab-v0', custom_cfgs=custom_cfgs) agent.learn() diff --git a/examples/train_sb3.py b/examples/train_sb3.py index 963e2c4..4868f56 100644 --- a/examples/train_sb3.py +++ b/examples/train_sb3.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # SB3 from stable_baselines3 import PPO from stable_baselines3.common.callbacks import BaseCallback diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4a84ecc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "livinglab" +version = "0.1.0" +description = "LivingLab: Intelligent Energy Management RL Environment" +readme = "README.md" +requires-python = ">=3.8" +authors = [{ name = "ISLa Lab" }] +dependencies = [ + "gymnasium>=0.28.1", + "numpy", + "pandas", + "ladybug-core", + "tabulate", + "ipywidgets", + "torch==2.8.0", # + "torchvision==0.23.0", # -> compatability with Omnisafe + "stable-baselines3==2.0.0", # + # Add any other required packages here +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["livinglab*", "ext*"] + +[tool.setuptools.package-data] +livinglab = ["*"] \ No newline at end of file diff --git a/scripts/agent.py b/scripts/agent.py index 2f9a5c2..57d2c70 100644 --- a/scripts/agent.py +++ b/scripts/agent.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # LivingLab from livinglab.envs.livinglab_env import LivingLabEnv diff --git a/scripts/eval.py b/scripts/eval.py index 1264d6a..f80624a 100644 --- a/scripts/eval.py +++ b/scripts/eval.py @@ -1,11 +1,9 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # LivingLab from livinglab.envs.livinglab_env import LivingLabEnv from livinglab.utils.wrappers import NormalizedSpaceWrapper # Utils +import os import wandb import json, yaml import argparse diff --git a/scripts/eval_rbc.py b/scripts/eval_rbc.py index a785aab..0f73d56 100644 --- a/scripts/eval_rbc.py +++ b/scripts/eval_rbc.py @@ -1,10 +1,8 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # LivingLab from livinglab.envs.livinglab_env import LivingLabEnv # Utils +import os import wandb import json, yaml import argparse diff --git a/scripts/train.py b/scripts/train.py index eb5f88b..f7c2c38 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # Omnisafe from ext import omnisafe @@ -8,6 +5,7 @@ from livinglab.envs.livinglab_env import LivingLabEnv # Utils +import os import torch import argparse import yaml, json From e87991ee36953d616e45c2e90558594a4f660ae3 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Thu, 18 Jun 2026 15:32:51 +0200 Subject: [PATCH 02/21] Updated README and gitignore --- .gitignore | 3 +++ README.md | 28 ++++++++++++---------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 3286494..f55b4f1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # Omnisafe runs **/runs/* +# Installation artifacts +**/*.egg-info/* + # Experiments and wandb **/tea_debug.log **/wandb/* diff --git a/README.md b/README.md index c24157c..d706983 100644 --- a/README.md +++ b/README.md @@ -50,29 +50,25 @@ We recommend using a [Miniconda](https://www.anaconda.com/docs/getting-started/m ```bash git clone https://github.com/Isla-lab/LivingLab.git ``` -2. Setup [Omnisafe](https://github.com/PKU-Alignment/omnisafe): +2. Create and activate a miniconda environment: ```bash - # 1. Create the conda environment - cd LivingLab/ext/omnisafe - conda env create --file conda-recipe.yaml - - # 2. Install omnisafe - conda activate safe-livinglab + conda create -n livinglab python==3.10 -y + conda activate livinglab + ``` +3. Setup `LivingLab`: + ```bash + cd LivingLab/ pip install -e . ``` -3. Update and install additional libraries +4. Setup [Omnisafe](https://github.com/PKU-Alignment/omnisafe): ```bash - # 1. Update torch and torchvision - pip install torch==2.8.0 torchvision==0.23.0 - - # 2. Install utilities - pip install ipywidgets - pip install ladybug-core - pip install stable_baselines3==2.0.0 + cd ext/omnisafe + conda activate safe-livinglab + pip install -e . ``` ## 2. 🚀 Quick Start -You can easily use the environment by importing it and and calling calling `.gym.make()`. +You can easily use the environment by importing it and and calling calling `gym.make()`. ```python import gymnasium as gym import livinglab # Registers the environment with default configurations From 757b98cee1d4e716c3b72d9f46b546e2bed900c1 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Wed, 24 Jun 2026 15:45:58 +0200 Subject: [PATCH 03/21] Fixed trin/eval scripts --- scripts/eval_rbc.py | 27 +++++++++++++++------------ scripts/train.py | 7 ++++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/scripts/eval_rbc.py b/scripts/eval_rbc.py index 0f73d56..b187085 100644 --- a/scripts/eval_rbc.py +++ b/scripts/eval_rbc.py @@ -113,9 +113,10 @@ def compare_temperature(args, results): bx1.plot(np.zeros_like(cooling_demand), color='black', linestyle='--') bx1.grid('on') - os.makedirs(f'./experiments/{args.agent}/figs', exist_ok=True) - fig.savefig(f'./experiments/{args.agent}/figs/indoor_dry_bulb_temperature.png', format='png') - plt.show() + name = args.agent if args.name is None else args.name + os.makedirs(f'./experiments/{name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/figs/indoor_dry_bulb_temperature.png', format='png') + # plt.show() def compare_hp_usage(args, results): # Outdoor temperatures @@ -162,9 +163,10 @@ def compare_hp_usage(args, results): bx.legend() bx.grid('on') - os.makedirs(f'./experiments/{args.agent}/figs', exist_ok=True) - fig.savefig(f'./experiments/{args.agent}/figs/mshp_usage.png', format='png') - plt.show() + name = args.agent if args.name is None else args.name + os.makedirs(f'./experiments/{name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/figs/mshp_usage.png', format='png') + # plt.show() def compare_thermal_battery(args, results): @@ -186,9 +188,10 @@ def compare_thermal_battery(args, results): bx2.set_ylim(ymin=-0.05, ymax=1.05) bx2.yaxis.label.set_color('xkcd:orange') - os.makedirs(f'./experiments/{args.agent}/figs', exist_ok=True) - fig.savefig(f'./experiments/{args.agent}/figs/thermal_battery.png', format='png') - plt.show() + name = args.agent if args.name is None else args.name + os.makedirs(f'./experiments/{name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/figs/thermal_battery.png', format='png') + # plt.show() def log_kpis(args, results): @@ -206,7 +209,7 @@ def log_kpis(args, results): run = wandb.init( entity=args.entity, project='LivingLab_RL_eval_v3' if args.project is None else args.project, - name=args.agent + name=args.agent if args.name is None else args.name ) # Log results @@ -289,5 +292,5 @@ def eval(args, env_cfgs): compare_temperature(args, results) compare_hp_usage(args, results) - # if args.wandb: - # log_kpis(args, results) \ No newline at end of file + if args.wandb: + log_kpis(args, results) \ No newline at end of file diff --git a/scripts/train.py b/scripts/train.py index f7c2c38..9d8121d 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -37,6 +37,7 @@ def parse_arguments(): parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") parser.add_argument('--project', type=str, nargs='?', help="Wandb project") parser.add_argument('--entity', type=str, nargs='?', help="Wandb entity") + parser.add_argument('--exp_name', type=str, nargs='?', help="Experiment name") parser.add_argument('--tag', type=str, nargs='*', help="Wandb tag") return parser.parse_args() @@ -86,7 +87,7 @@ def train(args, env_cfgs): 'wandb_project': args.project if args.project is not None else 'None', 'entity': args.entity if args.entity is not None else 'None', 'mode': 'online', - 'tag': list(args.tag) if args.tag is not None else [] + 'tag': [args.exp_name] }, # --- LIVINGLAB KEY ARGUMENTS --- @@ -121,8 +122,8 @@ def train(args, env_cfgs): args = parse_arguments() # Experiment logging - exp_name = datetime.now().strftime('%d-%m-%y_%H:%M') - exp_dir = f'./experiments/{args.algo}_{exp_name}' + exp_name = f"{args.algo}_{datetime.now().strftime('%d-%m-%y_%H:%M')}" if args.exp_name is None else args.exp_name + exp_dir = f'./experiments/{exp_name}' seed_dir = f'{exp_dir}/seed{args.seed}' os.makedirs(seed_dir, exist_ok=True) From 4a047d9912e14fdd08396a9a51b8c6768f4cdcc9 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Wed, 24 Jun 2026 15:46:36 +0200 Subject: [PATCH 04/21] Adjusted ComfortRBC controls to propagation --- scripts/agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/agent.py b/scripts/agent.py index 57d2c70..79c0a1f 100644 --- a/scripts/agent.py +++ b/scripts/agent.py @@ -137,11 +137,11 @@ def predict(self, observations: Mapping[str, Any]) -> np.ndarray: # Above the set-point if hot_delta > 0: if hot_delta > self.comfort_band: # <- Too hot - scheduled_actions[hp_idx] = 0.8*np.sign(scheduled_actions[hp_idx]) + scheduled_actions[hp_idx] = 0.35*np.sign(scheduled_actions[hp_idx]) if thm_soc > 0.1: scheduled_actions[thm_idx] = min(scheduled_actions[thm_idx], -thm_soc/2) else: # <- Hot within the band - scheduled_actions[hp_idx] = 0.2*np.sign(scheduled_actions[hp_idx]) + scheduled_actions[hp_idx] = 0.3*np.sign(scheduled_actions[hp_idx]) if thm_soc > 0.1: scheduled_actions[thm_idx] = min(scheduled_actions[thm_idx], -thm_soc/3) @@ -149,7 +149,7 @@ def predict(self, observations: Mapping[str, Any]) -> np.ndarray: else: temp_delta = outdoor_dry_bulb_temperature - indoor_dry_bulb_temperature if temp_delta > 0: # Outdoor temperature affects indoors - scheduled_actions[hp_idx] = 0.3*np.sign(scheduled_actions[hp_idx]) + scheduled_actions[hp_idx] = 0.15*np.sign(scheduled_actions[hp_idx]) else: scheduled_actions[hp_idx] = 0.0 From 82d61f04c4be0848e733e289ddb0cd663f1d71b1 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Wed, 24 Jun 2026 15:47:25 +0200 Subject: [PATCH 05/21] Added thermal propagation modes to LivingLabEnv --- livinglab/envs/livinglab_env.py | 66 ++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/livinglab/envs/livinglab_env.py b/livinglab/envs/livinglab_env.py index cfd0cd1..240c7b0 100644 --- a/livinglab/envs/livinglab_env.py +++ b/livinglab/envs/livinglab_env.py @@ -65,6 +65,7 @@ def __init__( active_observations: Optional[Iterable[str]]=[], inactive_observations: Optional[Iterable[str]]=[], periodic_observations_metadata: Optional[Mapping[str, Iterable[Union[int, float]]]]=None, + thermal_demand_propagation: Optional[int]=None, episode_length: Optional[int]=None, ): super().__init__(seed=seed, start_time_step=start_time_step, end_time_step=end_time_step, episode_length=episode_length) @@ -78,6 +79,7 @@ def __init__( # Options self.periodic_normalization = periodic_normalization self.periodic_observations_metadata = periodic_observations_metadata + self.thermal_demand_propagation = thermal_demand_propagation self.active_observations = set(active_observations) self.inactive_observations = set(inactive_observations) assert len(self.active_observations.intersection(inactive_observations)) == 0, \ @@ -246,6 +248,16 @@ def action_space(self) -> spaces.Box: """Environment's action space.""" return self._action_space + @property + def thermal_demand_propagation(self) -> int: + """ + Environment's mode of propagating the thermal demand generated at `self.time_step` to the next time step. + * 0 -> thermal demand affects the indoor dry-bulb temperature at `self.time_step` (no propagation) + * 1 -> thermal demand affects the indoor dry-bulb temperature at `self.time_step + 1` + * 2 -> thermal demand affects the indoor dry-bulb temperature at both `self.time_step` and `self.time_step + 1` + """ + return self._thermal_demand_propagation + @property def env_metadata(self) -> Mapping[str, Any]: return { @@ -325,6 +337,12 @@ def observation_space(self, new_space: spaces.Box): def action_space(self, new_space: spaces.Box): self._action_space = new_space + @thermal_demand_propagation.setter + def thermal_demand_propagation(self, new_mode: Optional[int]): + assert new_mode is None or new_mode in range(3), f'Invalid thermal demand propagation mode {new_mode}. Must be in [0, 1, 2].' + new_mode = 0 if new_mode is None else new_mode + self._thermal_demand_propagation = new_mode + def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tuple[np.ndarray, Mapping[str, Any]]: """ Reset `LivingLabEnv` to its initial state. @@ -398,9 +416,21 @@ def step(self, actions: Union[np.ndarray | List[float]]) -> Tuple[np.ndarray, fl self.apply_actions(**action_dict) # Update indoor temperature via dynamics - self._update_dynamics_input() - if self.simulate_dynamics: - self.update_indoor_dry_bulb_temperature() + if self._thermal_demand_propagation == 0: + self._extend_dynamics_input() + if self.simulate_dynamics: + self.update_indoor_dry_bulb_temperature() + else: + if self.simulate_dynamics: + if self._thermal_demand_propagation == 2: + self._update_last_dynamics_input( + vars={ + 'cooling_demand': self.energy_simulation.cooling_demand[self.time_step] + } + ) + self.update_indoor_dry_bulb_temperature() + else: + self._extend_dynamics_input() # Update environment variables (reflect effects of actions) net_electricity_consumption = ( @@ -421,6 +451,15 @@ def step(self, actions: Union[np.ndarray | List[float]]) -> Tuple[np.ndarray, fl # Advance to the next time step self._next_time_step() + # TODO - ASSUMPTION: propagate the last agent's action to the current state + if self.simulate_dynamics and self._thermal_demand_propagation > 0: + # Extract the last cooling demand (result of the agent's action) + self._extend_dynamics_input(past=True) + + # Update the indoor temperature at the current `self.time_step` using the current observations + # In this way the agent "sees" the propagation of its actions through time + self.update_indoor_dry_bulb_temperature() + return self.observations(), reward, self.terminated, self.truncated, self.info def load_device(self, device: Union[Device, Mapping[str, Any]], device_class: Callable) -> Device: @@ -884,7 +923,7 @@ def _get_observations_data(self, past: bool=True) -> Mapping[str, Union[int, flo past_t = max(self.time_step - 1, 0) if past else self.time_step ep_past_t = max(self.episode_time_step - 1, 0) if past else self.episode_time_step observations.update({ - 'cooling_demand': self.energy_simulation.cooling_demand[past_t] + abs(min(self.thermal_battery.energy_balance[ep_past_t], 0.0)), + 'cooling_demand': self.energy_simulation.cooling_demand[past_t], 'thermal_battery_soc': self.thermal_battery.soc[ep_past_t], 'net_electricity_consumption': self.net_electricity_consumption[ep_past_t] }) @@ -900,9 +939,9 @@ def _next_time_step(self): # Increate `self.time_step` Environment.step(self) - def _update_dynamics_input(self, sanity_check: bool=True): + def _extend_dynamics_input(self, sanity_check: bool=True, past: bool=False): # Get current observations - obs = self.observations(include_all=True, past=False, names=True) + obs = self.observations(include_all=True, past=past, names=True) if sanity_check: missing = [name for name in self.dynamics.input_observation_names if name not in obs.keys()] assert len(missing) == 0, f'Missing observations required by the dynamics: {missing}.' @@ -918,6 +957,21 @@ def _update_dynamics_input(self, sanity_check: bool=True): ) ] + def _update_last_dynamics_input(self, vars: Mapping[str, float], sanity_check: bool=True): + if sanity_check: + missing = [name for name in vars.keys() if name not in self.dynamics.input_observation_names] + assert len(missing) == 0, f'Trying to update variables missing in the dynamics input: {missing}.' + + for name, value in vars.items(): + idx = self.dynamics.input_observation_names.index(name) + + # Normalization values + min_ = self.dynamics.input_norm_min[idx] + max_ = self.dynamics.input_norm_max[idx] + + # Update last value in the dynamics input + self.dynamics.model_input[idx][-1] = (value - min_)/(max_ - min_) + def _get_dynamics_input(self) -> torch.Tensor: model_input = [] From 69d62328625eb8abbc35c291d81b2aadd106a5f0 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Wed, 24 Jun 2026 15:48:05 +0200 Subject: [PATCH 06/21] Updated configurations towards smi2real mocking --- ...ylearn_challenge_2023_Building1 copy.json} | 3 +- .../citylearn_challenge_2023_Building2.json | 159 ++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) rename config/{device_only_obs.json => citylearn_challenge_2023_Building1 copy.json} (98%) create mode 100644 config/citylearn_challenge_2023_Building2.json diff --git a/config/device_only_obs.json b/config/citylearn_challenge_2023_Building1 copy.json similarity index 98% rename from config/device_only_obs.json rename to config/citylearn_challenge_2023_Building1 copy.json index 00a1f7f..1abd5b0 100644 --- a/config/device_only_obs.json +++ b/config/citylearn_challenge_2023_Building1 copy.json @@ -43,7 +43,7 @@ "active": true }, "comfort_band": { - "active": false + "active": true }, "cooling_demand": { "active": false @@ -152,6 +152,7 @@ ] }, "periodic_normalization": true, + "thermal_demand_propagation": 1, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json new file mode 100644 index 0000000..68b2acc --- /dev/null +++ b/config/citylearn_challenge_2023_Building2.json @@ -0,0 +1,159 @@ +{ + "seed": 42, + "sim_data_paths": { + "energy_simulation": "/datasets/citylearn_challenge_2023_phase_1/Building_1.csv", + "weather": "/datasets/citylearn_challenge_2023_phase_1/weather.csv", + "pricing": "/datasets/citylearn_challenge_2023_phase_1/pricing.csv", + "carbon_intensity": "/datasets/citylearn_challenge_2023_phase_1/carbon_intensity.csv" + }, + "start_time_step": 0, + "end_time_step": 719, + "observations_metadata": { + "hour": { + "active": true, + "periodic_metadata": { + "min": 1, + "max": 24 + } + }, + "day_type": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 7 + } + }, + "month": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 12 + } + }, + "indoor_dry_bulb_temperature": { + "active": true + }, + "indoor_relative_humidity": { + "active": false + }, + "occupant_count": { + "active": false + }, + "indoor_dry_bulb_temperature_cooling_set_point": { + "active": true + }, + "comfort_band": { + "active": true + }, + "cooling_demand": { + "active": false + }, + "non_shiftable_load": { + "active": false + }, + "solar_generation": { + "active": false + }, + "outdoor_dry_bulb_temperature": { + "active": true + }, + "outdoor_relative_humidity": { + "active": false + }, + "diffuse_solar_irradiance": { + "active": false + }, + "direct_solar_irradiance": { + "active": false + }, + "electricity_pricing": { + "active": false + }, + "carbon_intensity": { + "active": false + }, + "thermal_battery_soc": { + "active": true + }, + "net_electricity_consumption": { + "active": false + }, + "underground_temperature": { + "active": true + } + }, + "heat_pump_cfgs": { + "nominal_power": 2.25, + "efficiency": 0.29, + "mode": "cooling", + "target_temperature": 6.0, + "tank_depth": 5.0, + "soil_alpha": 0.052, + "kasuda_data": "../data/datasets/citylearn_challenge_2023_phase_1/USA_TX_San.Antonio.Intl.AP.722530_TMY3.epw" + }, + "thermal_battery_cfgs": { + "capacity": 4.0, + "efficiency": 0.95, + "loss_coef": 1e-05 + }, + "pv_system_cfgs": { + "nominal_power": 1.2 + }, + "dynamics_cfgs": { + "path": "/datasets/citylearn_challenge_2023_phase_1/Building_1.pth", + "num_layers": 2, + "input_size": 13, + "hidden_size": 16, + "lookback": 12, + "input_observation_names": [ + "direct_solar_irradiance", + "diffuse_solar_irradiance", + "outdoor_dry_bulb_temperature", + "indoor_dry_bulb_temperature_cooling_set_point", + "occupant_count", + "cooling_demand", + "month_sin", + "month_cos", + "hour_sin", + "hour_cos", + "day_type_sin", + "day_type_cos", + "indoor_dry_bulb_temperature" + ], + "input_norm_min": [ + 0.0, + 0.0, + 21.7, + 18.88889, + 0.0, + 0.0, + -0.8660254, + -1.0, + -1.0, + -1.0, + -0.9749279, + -0.90096885, + 12.83158 + ], + "input_norm_max": [ + 931.0, + 486.5, + 42.8, + 24.444445, + 3.0, + 12.6740625, + 1.2246469000000002e-16, + -0.5, + 1.0, + 1.0, + 0.9749279, + 1.0, + 43.536755 + ] + }, + "periodic_normalization": true, + "thermal_demand_propagation": 1, + "reward_fn": { + "class": "ComfortRewardFunction" + } +} \ No newline at end of file From 9ba0a246275f84b0d74fd1fcad0d6c8d57501906 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Wed, 24 Jun 2026 15:48:25 +0200 Subject: [PATCH 07/21] Updated .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f55b4f1..a476154 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ # Installation artifacts **/*.egg-info/* +# Bash scripts +**/*.sh + # Experiments and wandb **/tea_debug.log **/wandb/* From c38ef7199a162fdf40a1a9c047ca058951cfba09 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Thu, 25 Jun 2026 15:34:27 +0200 Subject: [PATCH 08/21] Fixed JSON file configurations --- ...> citylearn_challenge_2023_Building1.json} | 0 .../citylearn_challenge_2023_Building2.json | 16 +- .../citylearn_challenge_2023_Building3.json | 159 ++++++++++++++++++ 3 files changed, 167 insertions(+), 8 deletions(-) rename config/{citylearn_challenge_2023_Building1 copy.json => citylearn_challenge_2023_Building1.json} (100%) create mode 100644 config/citylearn_challenge_2023_Building3.json diff --git a/config/citylearn_challenge_2023_Building1 copy.json b/config/citylearn_challenge_2023_Building1.json similarity index 100% rename from config/citylearn_challenge_2023_Building1 copy.json rename to config/citylearn_challenge_2023_Building1.json diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json index 68b2acc..40122a5 100644 --- a/config/citylearn_challenge_2023_Building2.json +++ b/config/citylearn_challenge_2023_Building2.json @@ -1,7 +1,7 @@ { "seed": 42, "sim_data_paths": { - "energy_simulation": "/datasets/citylearn_challenge_2023_phase_1/Building_1.csv", + "energy_simulation": "/datasets/citylearn_challenge_2023_phase_1/Building_2.csv", "weather": "/datasets/citylearn_challenge_2023_phase_1/weather.csv", "pricing": "/datasets/citylearn_challenge_2023_phase_1/pricing.csv", "carbon_intensity": "/datasets/citylearn_challenge_2023_phase_1/carbon_intensity.csv" @@ -100,7 +100,7 @@ "nominal_power": 1.2 }, "dynamics_cfgs": { - "path": "/datasets/citylearn_challenge_2023_phase_1/Building_1.pth", + "path": "/datasets/citylearn_challenge_2023_phase_1/Building_2.pth", "num_layers": 2, "input_size": 13, "hidden_size": 16, @@ -124,7 +124,7 @@ 0.0, 0.0, 21.7, - 18.88889, + 20.444445, 0.0, 0.0, -0.8660254, @@ -133,22 +133,22 @@ -1.0, -0.9749279, -0.90096885, - 12.83158 + 16.119997 ], "input_norm_max": [ 931.0, 486.5, 42.8, - 24.444445, - 3.0, - 12.6740625, + 26.666666, + 1.0, + 11.077426, 1.2246469000000002e-16, -0.5, 1.0, 1.0, 0.9749279, 1.0, - 43.536755 + 42.521355 ] }, "periodic_normalization": true, diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json new file mode 100644 index 0000000..3adc5f3 --- /dev/null +++ b/config/citylearn_challenge_2023_Building3.json @@ -0,0 +1,159 @@ +{ + "seed": 42, + "sim_data_paths": { + "energy_simulation": "/datasets/citylearn_challenge_2023_phase_1/Building_3.csv", + "weather": "/datasets/citylearn_challenge_2023_phase_1/weather.csv", + "pricing": "/datasets/citylearn_challenge_2023_phase_1/pricing.csv", + "carbon_intensity": "/datasets/citylearn_challenge_2023_phase_1/carbon_intensity.csv" + }, + "start_time_step": 0, + "end_time_step": 719, + "observations_metadata": { + "hour": { + "active": true, + "periodic_metadata": { + "min": 1, + "max": 24 + } + }, + "day_type": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 7 + } + }, + "month": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 12 + } + }, + "indoor_dry_bulb_temperature": { + "active": true + }, + "indoor_relative_humidity": { + "active": false + }, + "occupant_count": { + "active": false + }, + "indoor_dry_bulb_temperature_cooling_set_point": { + "active": true + }, + "comfort_band": { + "active": true + }, + "cooling_demand": { + "active": false + }, + "non_shiftable_load": { + "active": false + }, + "solar_generation": { + "active": false + }, + "outdoor_dry_bulb_temperature": { + "active": true + }, + "outdoor_relative_humidity": { + "active": false + }, + "diffuse_solar_irradiance": { + "active": false + }, + "direct_solar_irradiance": { + "active": false + }, + "electricity_pricing": { + "active": false + }, + "carbon_intensity": { + "active": false + }, + "thermal_battery_soc": { + "active": true + }, + "net_electricity_consumption": { + "active": false + }, + "underground_temperature": { + "active": true + } + }, + "heat_pump_cfgs": { + "nominal_power": 2.78, + "efficiency": 0.26, + "mode": "cooling", + "target_temperature": 6.21, + "tank_depth": 7.5, + "soil_alpha": 0.052, + "kasuda_data": "../data/datasets/citylearn_challenge_2023_phase_1/USA_TX_San.Antonio.Intl.AP.722530_TMY3.epw" + }, + "thermal_battery_cfgs": { + "capacity": 3.3, + "efficiency": 0.96, + "loss_coef": 1e-05 + }, + "pv_system_cfgs": { + "nominal_power": 2.4 + }, + "dynamics_cfgs": { + "path": "/datasets/citylearn_challenge_2023_phase_1/Building_3.pth", + "num_layers": 2, + "input_size": 13, + "hidden_size": 16, + "lookback": 12, + "input_observation_names": [ + "direct_solar_irradiance", + "diffuse_solar_irradiance", + "outdoor_dry_bulb_temperature", + "indoor_dry_bulb_temperature_cooling_set_point", + "occupant_count", + "cooling_demand", + "month_sin", + "month_cos", + "hour_sin", + "hour_cos", + "day_type_sin", + "day_type_cos", + "indoor_dry_bulb_temperature" + ], + "input_norm_min": [ + 0.0, + 0.0, + 21.7, + 21.666666, + 0.0, + 0.0, + -0.8660254, + -1.0, + -1.0, + -1.0, + -0.9749279, + -0.90096885, + 17.278326 + ], + "input_norm_max": [ + 931.0, + 486.5, + 42.8, + 29.444445, + 2.0, + 11.438825, + 1.2246469000000002e-16, + -0.5, + 1.0, + 1.0, + 0.9749279, + 1.0, + 43.541588 + ] + }, + "periodic_normalization": true, + "thermal_demand_propagation": 1, + "reward_fn": { + "class": "ComfortRewardFunction" + } +} \ No newline at end of file From 04afea5fec2e51684850a422615e52e14c95c41b Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Thu, 25 Jun 2026 15:35:13 +0200 Subject: [PATCH 09/21] Fixed episode length management when chainging start or end timestep --- livinglab/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/livinglab/base.py b/livinglab/base.py index 15bbcbc..5b49e46 100644 --- a/livinglab/base.py +++ b/livinglab/base.py @@ -101,6 +101,8 @@ def start_time_step(self, new_step: Optional[int]): f'Invalid simulation start/end steps (start={new_step} >= end={self._end_time_step}).' self._start_time_step = new_step + if hasattr(self, '_episode_length'): + self.episode_length = min(self._episode_length, (self._end_time_step - self._start_time_step) + 1) @end_time_step.setter def end_time_step(self, new_step: Optional[int]): @@ -110,6 +112,8 @@ def end_time_step(self, new_step: Optional[int]): f'Invalid simulation start/end steps (start={self._start_time_step} >= end={new_step}).' self._end_time_step = new_step + if hasattr(self, '_episode_length'): + self.episode_length = min(self._episode_length, (self._end_time_step - self._start_time_step) + 1) @episode_length.setter def episode_length(self, new_len: Optional[int]): From edb308787bcc30ec711bacbe30357d7400fff5ba Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Thu, 25 Jun 2026 15:35:53 +0200 Subject: [PATCH 10/21] Modified train and eval scrips --- scripts/eval.py | 45 ++++++++++++++++++++++++++++----------------- scripts/eval_rbc.py | 35 +++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 31 deletions(-) diff --git a/scripts/eval.py b/scripts/eval.py index f80624a..f27be4c 100644 --- a/scripts/eval.py +++ b/scripts/eval.py @@ -49,6 +49,9 @@ def parse_arguments(): # LivingLab configs parser.add_argument('--exp_dir', type=str, default='./experiments/PPOLag_net_consumption_reward_discomfort_cost', help="Path to the experiment to evaluate") + parser.add_argument('--env_cfgs', type=str, nargs='?', help="Path to the JSON config file") + parser.add_argument('--start', type=int, nargs='?', help="Initial simulation time step") + parser.add_argument('--end', type=int, nargs='?', help="Ending simulation time step") # Wandb logging parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") @@ -124,9 +127,9 @@ def compare_temperature(args, results): bx1.plot(np.zeros_like(avg_demand), color='black', linestyle='--') bx1.grid('on') - os.makedirs(f'{args.exp_dir}/figs', exist_ok=True) - fig.savefig(f'{args.exp_dir}/figs/indoor_dry_bulb_temperature.png', format='png') - plt.show() + env_name = 'test' if args.env_cfgs is None else args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'{args.exp_dir}/figs/{env_name}', exist_ok=True) + fig.savefig(f'{args.exp_dir}/figs/{env_name}/indoor_dry_bulb_temperature.png', format='png') def compare_hp_usage(args, results): @@ -183,9 +186,9 @@ def compare_hp_usage(args, results): bx.legend() bx.grid('on') - os.makedirs(f'{args.exp_dir}/figs', exist_ok=True) - fig.savefig(f'{args.exp_dir}/figs/mshp_usage.png', format='png') - plt.show() + env_name = 'test' if args.env_cfgs is None else args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'{args.exp_dir}/figs/{env_name}', exist_ok=True) + fig.savefig(f'{args.exp_dir}/figs/{env_name}/mshp_usage.png', format='png') def compare_thermal_battery(args, results): @@ -208,10 +211,10 @@ def compare_thermal_battery(args, results): bx2.set_ylabel('SoC [%]') bx2.set_ylim(ymin=-0.05, ymax=1.05) bx2.yaxis.label.set_color('xkcd:orange') - - os.makedirs(f'{args.exp_dir}/figs', exist_ok=True) - fig.savefig(f'{args.exp_dir}/figs/thermal_battery.png', format='png') - plt.show() + + env_name = 'test' if args.env_cfgs is None else args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'{args.exp_dir}/figs/{env_name}', exist_ok=True) + fig.savefig(f'{args.exp_dir}/figs/{env_name}/thermal_battery.png', format='png') def log_kpis(args, results): @@ -246,9 +249,21 @@ def log_kpis(args, results): run.finish() -def eval(args, env_cfgs, seed): +def eval(args, seed): # Load the LivingLabEnv given the configurations - env = LivingLabEnv(**env_cfgs) + if args.env_cfgs is None: + with open(f'{args.exp_dir}/env_cfgs/test_cfgs.json', 'r') as f: + env_cfgs = json.load(f) + env = LivingLabEnv(**env_cfgs) + else: + env = LivingLabEnv.from_json(config=args.env_cfgs) + + # Modify start and end time step if provided + if args.start is not None: + env.start_time_step = args.start + if args.end is not None: + env.end_time_step = args.end + env = NormalizedSpaceWrapper(env) # Load the agent @@ -306,13 +321,9 @@ def eval(args, env_cfgs, seed): if __name__ == '__main__': args = parse_arguments() - # Load the test configurations - with open(f'{args.exp_dir}/env_cfgs/test_cfgs.json', 'r') as f: - env_cfgs = json.load(f) - results = defaultdict(dict) for i in range(1, len(glob(f'{args.exp_dir}/seed*'))+1): - results[f'seed{i}'] = eval(args, env_cfgs, seed=i) + results[f'seed{i}'] = eval(args, seed=i) compare_temperature(args, results) compare_hp_usage(args, results) diff --git a/scripts/eval_rbc.py b/scripts/eval_rbc.py index b187085..346415d 100644 --- a/scripts/eval_rbc.py +++ b/scripts/eval_rbc.py @@ -4,12 +4,10 @@ # Utils import os import wandb -import json, yaml +import json import argparse import numpy as np from matplotlib import pyplot as plt -from collections import defaultdict -from glob import glob from tabulate import tabulate from agent import HourRBC, ComfortRBC @@ -48,7 +46,9 @@ def parse_arguments(): # LivingLab configs parser.add_argument('--agent', type=str, default='HourRBC', help="RBC agent to evaluete") - parser.add_argument('--env_cfgs', type=str, default='./experiments/PPO_comfort_reward/env_cfgs/test_cfgs.json', help="Path to the configurations of the environment") + parser.add_argument('--env_cfgs', type=str, default='./config/default.json', help="Path to the configurations of the environment") + parser.add_argument('--start', type=int, nargs='?', help="Initial simulation time step") + parser.add_argument('--end', type=int, nargs='?', help="Ending simulation time step") # Wandb logging parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") @@ -114,9 +114,10 @@ def compare_temperature(args, results): bx1.grid('on') name = args.agent if args.name is None else args.name - os.makedirs(f'./experiments/{name}/figs', exist_ok=True) - fig.savefig(f'./experiments/{name}/figs/indoor_dry_bulb_temperature.png', format='png') - # plt.show() + env_name = args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'./experiments/{name}/{env_name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/{env_name}/figs/indoor_dry_bulb_temperature.png', format='png') + def compare_hp_usage(args, results): # Outdoor temperatures @@ -164,9 +165,9 @@ def compare_hp_usage(args, results): bx.grid('on') name = args.agent if args.name is None else args.name - os.makedirs(f'./experiments/{name}/figs', exist_ok=True) - fig.savefig(f'./experiments/{name}/figs/mshp_usage.png', format='png') - # plt.show() + env_name = args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'./experiments/{name}/{env_name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/{env_name}/figs/mshp_usage.png', format='png') def compare_thermal_battery(args, results): @@ -189,9 +190,9 @@ def compare_thermal_battery(args, results): bx2.yaxis.label.set_color('xkcd:orange') name = args.agent if args.name is None else args.name - os.makedirs(f'./experiments/{name}/figs', exist_ok=True) - fig.savefig(f'./experiments/{name}/figs/thermal_battery.png', format='png') - # plt.show() + env_name = args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'./experiments/{name}/{env_name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/{env_name}/figs/thermal_battery.png', format='png') def log_kpis(args, results): @@ -222,7 +223,13 @@ def log_kpis(args, results): def eval(args, env_cfgs): # Load the LivingLabEnv given the configurations - env = LivingLabEnv(**env_cfgs) + env = LivingLabEnv.from_json(config=env_cfgs) + + # Modify start and end time step if provided + if args.start is not None: + env.start_time_step = args.start + if args.end is not None: + env.end_time_step = args.end # Load the agent if args.agent == 'HourRBC': From aad02a0ed77b3c2dfd9dc461c30a058b3e852b7c Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Fri, 26 Jun 2026 09:25:29 +0200 Subject: [PATCH 11/21] Fixed typo in README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d706983..85a8bfb 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,6 @@ We recommend using a [Miniconda](https://www.anaconda.com/docs/getting-started/m 4. Setup [Omnisafe](https://github.com/PKU-Alignment/omnisafe): ```bash cd ext/omnisafe - conda activate safe-livinglab pip install -e . ``` From c6db3e10291babd2286ebe36b1d8fe7b030c791a Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Fri, 26 Jun 2026 09:30:50 +0200 Subject: [PATCH 12/21] Updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a476154..2899912 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # Installation artifacts **/*.egg-info/* +**/.vscode/* # Bash scripts **/*.sh From 6d0fc7692ae8d38be34e034641be79a9401035f6 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Fri, 26 Jun 2026 11:02:58 +0200 Subject: [PATCH 13/21] Added option to avoid to reset the dynamics input at each episode start --- .../citylearn_challenge_2023_Building1.json | 3 ++- .../citylearn_challenge_2023_Building2.json | 3 ++- .../citylearn_challenge_2023_Building3.json | 3 ++- livinglab/components/dynamics.py | 19 ++++++++++++++++--- livinglab/envs/livinglab_env.py | 8 +++++++- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/config/citylearn_challenge_2023_Building1.json b/config/citylearn_challenge_2023_Building1.json index 1abd5b0..9874687 100644 --- a/config/citylearn_challenge_2023_Building1.json +++ b/config/citylearn_challenge_2023_Building1.json @@ -149,7 +149,8 @@ 0.9749279, 1.0, 43.536755 - ] + ], + "reset_on_ep_start": true }, "periodic_normalization": true, "thermal_demand_propagation": 1, diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json index 40122a5..9fd9a4b 100644 --- a/config/citylearn_challenge_2023_Building2.json +++ b/config/citylearn_challenge_2023_Building2.json @@ -149,7 +149,8 @@ 0.9749279, 1.0, 42.521355 - ] + ], + "reset_on_ep_start": true }, "periodic_normalization": true, "thermal_demand_propagation": 1, diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json index 3adc5f3..126ab82 100644 --- a/config/citylearn_challenge_2023_Building3.json +++ b/config/citylearn_challenge_2023_Building3.json @@ -149,7 +149,8 @@ 0.9749279, 1.0, 43.541588 - ] + ], + "reset_on_ep_start": true }, "periodic_normalization": true, "thermal_demand_propagation": 1, diff --git a/livinglab/components/dynamics.py b/livinglab/components/dynamics.py index fb9744f..054404d 100644 --- a/livinglab/components/dynamics.py +++ b/livinglab/components/dynamics.py @@ -3,7 +3,7 @@ from pathlib import Path from abc import ABC, abstractmethod -from typing import Tuple, List, Union +from typing import Optional, Tuple, List, Union class Dynamics(ABC): """ @@ -30,8 +30,9 @@ def __init__( num_layers: int, hidden_size: int, lookback: int, - input_size: int=None, - dropout: float=0.0 + input_size: Optional[int]=None, + dropout: float=0.0, + reset_on_ep_start: Optional[bool]=None ): Dynamics.__init__(self) nn.Module.__init__(self) @@ -61,6 +62,9 @@ def __init__( in_features=self.hidden_size, out_features=1, # <- the predicted indoor temperature ) + + # Dynamics options + self.reset_on_ep_start = reset_on_ep_start @property def model_input(self): @@ -82,6 +86,10 @@ def hidden_state(self): def input_size(self): return self._input_size + @property + def reset_on_ep_start(self) -> bool: + return self._reset_on_ep_start + @input_size.setter def input_size(self, new_size: int): assert new_size is None or new_size > 0, f'Invalid input dimensionality {new_size}. Must be either `None` or > 0.' @@ -95,6 +103,11 @@ def model_input(self, new_input): def hidden_state(self, new_h: Tuple[torch.Tensor, torch.Tensor]): self._hidden_state = new_h + @reset_on_ep_start.setter + def reset_on_ep_start(self, new_val: Optional[bool]): + assert new_val is None or isinstance(new_val, bool), f'Invalid type for `LSTMDynamics.reset_on_ep_start`. Required `bool`, found {type(new_val)}.' + self._reset_on_ep_start = True if new_val is None else new_val + def forward(self, x: torch.Tensor, h: Tuple[torch.Tensor, torch.Tensor]) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: lstm_out, h = self.l_lstm(x, h) lstm_out = self.dropout(lstm_out) diff --git a/livinglab/envs/livinglab_env.py b/livinglab/envs/livinglab_env.py index 240c7b0..000d481 100644 --- a/livinglab/envs/livinglab_env.py +++ b/livinglab/envs/livinglab_env.py @@ -258,6 +258,11 @@ def thermal_demand_propagation(self) -> int: """ return self._thermal_demand_propagation + @property + def reset_dynamics(self) -> bool: + """Whether to call `self.dynamics.reset()` upon calling `self.reset()`.""" + return self.episode_counter < 1 or self.dynamics.reset_on_ep_start + @property def env_metadata(self) -> Mapping[str, Any]: return { @@ -376,7 +381,8 @@ def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tu self.pv_system.reset() # Reset dynamics - self.dynamics.reset() + if self.reset_dynamics: + self.dynamics.reset() # Reset additional variables self._episode_rewards = np.zeros(self.episode_length, dtype=np.float32) From 3f0dbc16593ffcdf75d74202d5aeae8d8a528ac7 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Fri, 26 Jun 2026 11:03:46 +0200 Subject: [PATCH 14/21] Included a safe reset property --- livinglab/base.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/livinglab/base.py b/livinglab/base.py index 5b49e46..0bc4e16 100644 --- a/livinglab/base.py +++ b/livinglab/base.py @@ -32,6 +32,7 @@ def __init__(self, seed: Optional[int]=None, start_time_step: Optional[int]=None # Episodic info self.episode_length = episode_length + self._ready_to_reset = True @property def seed(self) -> int: @@ -58,6 +59,11 @@ def episode_length(self): """Time steps duration of a simulation episode.""" return self._episode_length + @property + def ready_to_reset(self) -> bool: + """Wheather the environment can safely call `Environment.reset()`""" + return self._ready_to_reset + @property def simulation_episodes(self): """Number of simulation episodes.""" @@ -134,6 +140,10 @@ def episode_length(self, new_len: Optional[int]): self.simulation_episodes = (self._end_time_step + 1) // self.episode_length self.episode_counter = -1 + @ready_to_reset.setter + def ready_to_reset(self, new_val: bool): + self._ready_to_reset = new_val + @simulation_episodes.setter def simulation_episodes(self, n: int): assert n > 0, f'Invalid number of simulation episodes n={n}. Must be > 0.' @@ -164,16 +174,19 @@ def episode_end_time_step(self, new_step: int): def step(self): self.episode_time_step += 1 + self._ready_to_reset = True def reset(self): """ Reset the environment to its initial state, and set next `self.episode_start_time_step` and `self.episode_end_time_step` correclty. """ - self.episode_counter += 1 - self.episode_time_step = 0 - self.episode_start_time_step = self.start_time_step + (self.episode_counter % self.simulation_episodes) * self.episode_length - self.episode_end_time_step = self.episode_start_time_step + self.episode_length - 1 + if self.ready_to_reset: + self.episode_counter += 1 + self.episode_time_step = 0 + self.episode_start_time_step = self.start_time_step + (self.episode_counter % self.simulation_episodes) * self.episode_length + self.episode_end_time_step = self.episode_start_time_step + self.episode_length - 1 + self._ready_to_reset = False class Device(Environment): From 8cbdb10b3e1839ddd17d9736c452c9eab5406c6f Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Fri, 26 Jun 2026 12:06:19 +0200 Subject: [PATCH 15/21] Added option for starting an episode at a random valid time step --- .../citylearn_challenge_2023_Building1.json | 1 + .../citylearn_challenge_2023_Building2.json | 1 + .../citylearn_challenge_2023_Building3.json | 1 + livinglab/base.py | 10 +++++-- livinglab/components/battery.py | 13 +++++++-- livinglab/components/device.py | 13 +++++++-- livinglab/envs/livinglab_env.py | 28 ++++++++++++++++--- 7 files changed, 55 insertions(+), 12 deletions(-) diff --git a/config/citylearn_challenge_2023_Building1.json b/config/citylearn_challenge_2023_Building1.json index 9874687..bd9bfc3 100644 --- a/config/citylearn_challenge_2023_Building1.json +++ b/config/citylearn_challenge_2023_Building1.json @@ -154,6 +154,7 @@ }, "periodic_normalization": true, "thermal_demand_propagation": 1, + "random_ep_reset": false, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json index 9fd9a4b..892a2b8 100644 --- a/config/citylearn_challenge_2023_Building2.json +++ b/config/citylearn_challenge_2023_Building2.json @@ -154,6 +154,7 @@ }, "periodic_normalization": true, "thermal_demand_propagation": 1, + "random_ep_reset": false, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json index 126ab82..cfe0255 100644 --- a/config/citylearn_challenge_2023_Building3.json +++ b/config/citylearn_challenge_2023_Building3.json @@ -154,6 +154,7 @@ }, "periodic_normalization": true, "thermal_demand_propagation": 1, + "random_ep_reset": false, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/livinglab/base.py b/livinglab/base.py index 0bc4e16..3f6e84d 100644 --- a/livinglab/base.py +++ b/livinglab/base.py @@ -176,15 +176,21 @@ def step(self): self.episode_time_step += 1 self._ready_to_reset = True - def reset(self): + def reset(self, episode_start_time_step: Optional[int]=None): """ Reset the environment to its initial state, and set next `self.episode_start_time_step` and `self.episode_end_time_step` correclty. + + Parameters + ---------- + :param episode_start_time_step: Explicit time step to reset the environment to + :type episode_start_time_step: int """ if self.ready_to_reset: self.episode_counter += 1 self.episode_time_step = 0 - self.episode_start_time_step = self.start_time_step + (self.episode_counter % self.simulation_episodes) * self.episode_length + self.episode_start_time_step = episode_start_time_step if episode_start_time_step is not None else \ + self.start_time_step + (self.episode_counter % self.simulation_episodes) * self.episode_length self.episode_end_time_step = self.episode_start_time_step + self.episode_length - 1 self._ready_to_reset = False diff --git a/livinglab/components/battery.py b/livinglab/components/battery.py index 22bde43..a258ad6 100644 --- a/livinglab/components/battery.py +++ b/livinglab/components/battery.py @@ -155,9 +155,16 @@ def charge(self, energy: float): else: self._energy_balance[self.episode_time_step] = delta_energy * self.round_trip_efficiency - def reset(self): - """Reset the Thermal Battery to its initial state.""" - super().reset() + def reset(self, **kwargs: Mapping[str, Any]): + """ + Reset the Thermal Battery to its initial state. + + Parameters + ---------- + :param kwargs: Keyword parameters for `super().reset()` + :type kwargs: Mapping[str, Any] + """ + super().reset(**kwargs) self._soc = np.zeros(self.episode_length, dtype=np.float32) self._soc[0] = self.initial_soc self._energy_balance = np.zeros(self.episode_length, dtype=np.float32) diff --git a/livinglab/components/device.py b/livinglab/components/device.py index 79b6771..746eafa 100644 --- a/livinglab/components/device.py +++ b/livinglab/components/device.py @@ -57,9 +57,16 @@ def update_electricity_consumption(self, electricity_consumption: float, enforce f'Invalid electricity consumption value {electricity_consumption}. Must be >= 0.' self._electricity_consumption[self.episode_time_step] += electricity_consumption - def reset(self): - """Reset the Electric Device to its initial state.""" - super().reset() + def reset(self, **kwargs: Mapping[str, Any]): + """ + Reset the Electric Device to its initial state. + + Parameters + ---------- + :param kwargs: Keyword parameters for `super().reset()` + :type kwargs: Mapping[str, Any] + """ + super().reset(**kwargs) self._electricity_consumption = np.zeros(self.episode_length, dtype=np.float32) def get_metadata(self) -> Mapping[str, Any]: diff --git a/livinglab/envs/livinglab_env.py b/livinglab/envs/livinglab_env.py index 000d481..7735b12 100644 --- a/livinglab/envs/livinglab_env.py +++ b/livinglab/envs/livinglab_env.py @@ -66,6 +66,7 @@ def __init__( inactive_observations: Optional[Iterable[str]]=[], periodic_observations_metadata: Optional[Mapping[str, Iterable[Union[int, float]]]]=None, thermal_demand_propagation: Optional[int]=None, + random_ep_reset: Optional[bool]=None, episode_length: Optional[int]=None, ): super().__init__(seed=seed, start_time_step=start_time_step, end_time_step=end_time_step, episode_length=episode_length) @@ -80,6 +81,7 @@ def __init__( self.periodic_normalization = periodic_normalization self.periodic_observations_metadata = periodic_observations_metadata self.thermal_demand_propagation = thermal_demand_propagation + self.random_ep_reset = random_ep_reset self.active_observations = set(active_observations) self.inactive_observations = set(inactive_observations) assert len(self.active_observations.intersection(inactive_observations)) == 0, \ @@ -258,6 +260,11 @@ def thermal_demand_propagation(self) -> int: """ return self._thermal_demand_propagation + @property + def random_ep_reset(self) -> bool: + """Whether to restart each episode to a random initial time step.""" + return self._random_ep_reset + @property def reset_dynamics(self) -> bool: """Whether to call `self.dynamics.reset()` upon calling `self.reset()`.""" @@ -348,6 +355,11 @@ def thermal_demand_propagation(self, new_mode: Optional[int]): new_mode = 0 if new_mode is None else new_mode self._thermal_demand_propagation = new_mode + @random_ep_reset.setter + def random_ep_reset(self, new_val: Optional[bool]): + assert new_val is None or isinstance(new_val, bool), f'Invalid type for `LivingLabEnv.random_ep_reset`. Required `bool`, found {type(new_val)}.' + self._random_ep_reset = False if new_val is None else new_val + def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tuple[np.ndarray, Mapping[str, Any]]: """ Reset `LivingLabEnv` to its initial state. @@ -365,7 +377,15 @@ def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tu :rtype: np.ndarray """ gym.Env.reset(self) - Environment.reset(self) + + # Reset the environment to a valid random start time step + if self.random_ep_reset: + random_start_time_step = np.random.choice( + np.arange(self.start_time_step, self.end_time_step, self.episode_length) + ) + else: + random_start_time_step = None + Environment.reset(self, episode_start_time_step=random_start_time_step) # Check options if options is None: @@ -376,9 +396,9 @@ def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tu self.seed = seed # Reset devices - self.heat_pump.reset() - self.thermal_battery.reset() - self.pv_system.reset() + self.heat_pump.reset(episode_start_time_step=random_start_time_step) + self.thermal_battery.reset(episode_start_time_step=random_start_time_step) + self.pv_system.reset(episode_start_time_step=random_start_time_step) # Reset dynamics if self.reset_dynamics: From a7b9479699152f7f2a2cb7c212aee44378f6b777 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Sun, 28 Jun 2026 16:11:23 +0200 Subject: [PATCH 16/21] Added rendering option (TODO: real-time rendering) --- .../citylearn_challenge_2023_Building1.json | 4 + .../citylearn_challenge_2023_Building2.json | 4 + .../citylearn_challenge_2023_Building3.json | 4 + livinglab/envs/livinglab_env.py | 90 +++++++++++++++- livinglab/utils/functions.py | 101 ++++++++++++++++++ 5 files changed, 202 insertions(+), 1 deletion(-) diff --git a/config/citylearn_challenge_2023_Building1.json b/config/citylearn_challenge_2023_Building1.json index bd9bfc3..6515a71 100644 --- a/config/citylearn_challenge_2023_Building1.json +++ b/config/citylearn_challenge_2023_Building1.json @@ -155,6 +155,10 @@ "periodic_normalization": true, "thermal_demand_propagation": 1, "random_ep_reset": false, + "render_cfgs": { + "mode": "off", + "dir": "../render" + }, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json index 892a2b8..fbed0ba 100644 --- a/config/citylearn_challenge_2023_Building2.json +++ b/config/citylearn_challenge_2023_Building2.json @@ -155,6 +155,10 @@ "periodic_normalization": true, "thermal_demand_propagation": 1, "random_ep_reset": false, + "render_cfgs": { + "mode": "off", + "dir": "./render" + }, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json index cfe0255..488b65b 100644 --- a/config/citylearn_challenge_2023_Building3.json +++ b/config/citylearn_challenge_2023_Building3.json @@ -155,6 +155,10 @@ "periodic_normalization": true, "thermal_demand_propagation": 1, "random_ep_reset": false, + "render_cfgs": { + "mode": "off", + "dir": "./render" + }, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/livinglab/envs/livinglab_env.py b/livinglab/envs/livinglab_env.py index 7735b12..cac9b15 100644 --- a/livinglab/envs/livinglab_env.py +++ b/livinglab/envs/livinglab_env.py @@ -1,10 +1,14 @@ import torch import numpy as np import gymnasium as gym +import imageio.v2 as imageio from gymnasium import spaces +from PIL import Image +from matplotlib import pyplot as plt import os import json +from glob import glob from pathlib import Path from typing import Any, Literal, Optional, Union, Callable, Iterable, Tuple, Set, List, Mapping, Dict from typing_extensions import Self @@ -13,7 +17,7 @@ from livinglab.components.dynamics import Dynamics, LSTMDynamics from livinglab.components.device import DualSourceHeatPump, PVSystem from livinglab.components.battery import ThermalBattery -from livinglab.utils.functions import DAYS_PER_MONTH, CostFunctions +from livinglab.utils.functions import DAYS_PER_MONTH, UtilsFunctions, CostFunctions from livinglab.utils.data_loader import EnergySimulation, Weather, Pricing, CarbonEmissions from livinglab.utils.preprocessing import Normalize, PeriodicNormalization from livinglab.utils import rewards @@ -67,6 +71,7 @@ def __init__( periodic_observations_metadata: Optional[Mapping[str, Iterable[Union[int, float]]]]=None, thermal_demand_propagation: Optional[int]=None, random_ep_reset: Optional[bool]=None, + render_cfgs: Optional[Mapping[str, Any]]=None, episode_length: Optional[int]=None, ): super().__init__(seed=seed, start_time_step=start_time_step, end_time_step=end_time_step, episode_length=episode_length) @@ -82,6 +87,8 @@ def __init__( self.periodic_observations_metadata = periodic_observations_metadata self.thermal_demand_propagation = thermal_demand_propagation self.random_ep_reset = random_ep_reset + self.render_mode = None if render_cfgs is None else render_cfgs.get('mode', 'off') + self.render_dir = None if render_cfgs is None else render_cfgs.get('dir', None) self.active_observations = set(active_observations) self.inactive_observations = set(inactive_observations) assert len(self.active_observations.intersection(inactive_observations)) == 0, \ @@ -265,6 +272,20 @@ def random_ep_reset(self) -> bool: """Whether to restart each episode to a random initial time step.""" return self._random_ep_reset + @property + def render_mode(self) -> str: + """ + Living Lab render mode. + * `off` -> no render + * `on` -> render figures are saved to `self.render_dir` + * `real` -> real-time render + saving figures + """ + return self._render_mode + + @property + def render_dir(self) -> Union[str, None]: + return self._render_dir + @property def reset_dynamics(self) -> bool: """Whether to call `self.dynamics.reset()` upon calling `self.reset()`.""" @@ -360,6 +381,21 @@ def random_ep_reset(self, new_val: Optional[bool]): assert new_val is None or isinstance(new_val, bool), f'Invalid type for `LivingLabEnv.random_ep_reset`. Required `bool`, found {type(new_val)}.' self._random_ep_reset = False if new_val is None else new_val + @render_mode.setter + def render_mode(self, new_mode: Optional[str]): + assert new_mode is None or new_mode in ['off', 'on', 'real'], f'Invalid render mode {new_mode}. Must be in [`off`, `on`, `real`].' + + # TODO: work on real-time rendering + if new_mode == 'real': + print('[WARN] Real-time rendering has not been implemented, yet. Switched to render_mode=`on`.') + + self._render_mode = 'off' if new_mode is None else new_mode + + @render_dir.setter + def render_dir(self, new_dir: Optional[str]): + self._render_dir = '../render' if new_dir is None else new_dir + os.makedirs(self._render_dir, exist_ok=True) + def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tuple[np.ndarray, Mapping[str, Any]]: """ Reset `LivingLabEnv` to its initial state. @@ -486,8 +522,60 @@ def step(self, actions: Union[np.ndarray | List[float]]) -> Tuple[np.ndarray, fl # In this way the agent "sees" the propagation of its actions through time self.update_indoor_dry_bulb_temperature() + # TODO: render + if self.render_mode != 'off': + self.render() + return self.observations(), reward, self.terminated, self.truncated, self.info + def render(self): + # Retrieve data + indoor_dry_bulb_temperature = self.energy_simulation.indoor_dry_bulb_temperature[self.episode_start_time_step:self.time_step] + indoor_temperature_setpoint = self.energy_simulation.indoor_dry_bulb_temperature_cooling_set_point[self.episode_start_time_step:self.episode_end_time_step+1] + comfort_band = self.energy_simulation.comfort_band[self.episode_start_time_step:self.episode_end_time_step+1] + outdoor_dry_bulb_temperature = self.weather.outdoor_dry_bulb_temperature[self.episode_start_time_step:self.time_step] + cooling_demand = self.energy_simulation.cooling_demand[self.episode_start_time_step:self.time_step] + energy_from_battery = self.thermal_battery.energy_balance[:self.episode_time_step] + + # Create figure + fig = plt.figure(figsize=[20,10]) + + # Indoor dry-bulb temperature + ax = fig.add_subplot(2,1,1) + UtilsFunctions.render_indoor_state( + ax=ax, + indoor_dry_bulb_temperature=indoor_dry_bulb_temperature, + indoor_temperature_setpoint=indoor_temperature_setpoint, + comfort_band=comfort_band, + outdoor_dry_bulb_temperature=outdoor_dry_bulb_temperature, + time_steps=self.episode_length + ) + + # Thermal demand and Battery energy balance + bx = fig.add_subplot(2,1,2) + UtilsFunctions.render_device_control( + ax=bx, + thermal_demand=cooling_demand, + energy_from_battery=energy_from_battery, + time_steps=self.episode_length + ) + + # Save the figure + os.makedirs(f'{self.render_dir}/ep{self.episode_counter:04}', exist_ok=True) + fig.savefig(f'{self.render_dir}/ep{self.episode_counter:04}/step{self.episode_time_step:03}.png', format='png') + plt.close(fig=fig) + + # Create GIF animation + if self.terminated: + image_files = sorted(glob(f'{self.render_dir}/ep{self.episode_counter:04}/*.png')) + imgs = [Image.open(f).convert('P', palette=Image.ADAPTIVE) for f in image_files] + imgs[0].save( + f'{self.render_dir}/ep{self.episode_counter:04}/animation.gif', + append_images=imgs[1:], + duration=100, + loop=0 + ) + def load_device(self, device: Union[Device, Mapping[str, Any]], device_class: Callable) -> Device: """ Load a simulation-ready device. diff --git a/livinglab/utils/functions.py b/livinglab/utils/functions.py index 75e4e84..87e2fb8 100644 --- a/livinglab/utils/functions.py +++ b/livinglab/utils/functions.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +from matplotlib.axes import Axes from ladybug.epw import EPW from typing import Optional, Union, Tuple, List, Mapping @@ -85,6 +86,106 @@ def extract_kasuda_parameters(path: str) -> Mapping[str, Union[int, float]]: return {'mean': t_mean, 'amplitude': amplitude, 't0': min_day} + @staticmethod + def render_indoor_state( + ax: Axes, + indoor_dry_bulb_temperature: np.ndarray, + indoor_temperature_setpoint: np.ndarray, + comfort_band: np.ndarray, + outdoor_dry_bulb_temperature: np.ndarray, + time_steps: int, + ): + """ + Render the current indoor state fo the Living Lab. + + Parameters + ---------- + :param ax: `matplotlib.Axes` + :type: Axes + :param indoor_dry_bulb_temperature: Current history of the indoor dry-bulb temperature + :type indoor_dry_bulb_temperature: np.ndarray + :param indoor_temperature_setpoint: User-defined setpoint throughout simulation. + :type indoor_temperature_setpoint: np.ndarray + :param comfort_band: Maximum deviation from the setpoint in terms of degrees to define comfort. + :type comfort_band: np.ndarray + :param outdoor_dry_bulb_temperature: Current history of the outdoor dry-bulb temperature + :type outdoor_dry_bulb_temperature: np.ndarray + :param time_steps: Simulation length + :type time_steps: int + """ + # Setpoint and comfort band + ax.fill_between( + range(time_steps), + indoor_temperature_setpoint + comfort_band, + indoor_temperature_setpoint - comfort_band, + color='g', + alpha=0.15, + label='Comfort band', + ) + + # Current indoor/outoor dry-bulb temperature + ax.plot(range(len(indoor_dry_bulb_temperature)), indoor_dry_bulb_temperature, linewidth=2.0, label='Indoor Dry Bulb Temperature', color='orange') + ax.plot(range(len(outdoor_dry_bulb_temperature)), outdoor_dry_bulb_temperature, label='Outdoor Dry Bulb Temperature', color='xkcd:light purple') + + # Style + ax.grid('on') + ax.set_title('Indoor Dry-bulb Temperature Evolution', fontweight='bold') + ax.set_ylabel('Temperature [°C]') + ax.legend(loc='upper left') + + @staticmethod + def render_device_control(ax: Axes, thermal_demand: np.ndarray, energy_from_battery: np.ndarray, time_steps: int): + """ + Render the current device control. + + Parameters + ---------- + :param ax: `matplotlib.Axes` + :type: Axes + :param thermal_demand: Current thermal demand history due to MSHP control. + :type thermal_demand: np.ndarray + :param energy_from_battery: Current thermal battery energy balance evolution. + :type energy_from_battery: np.ndarray + :param time_steps: Simulation length + :type time_steps: int + """ + + def _align_yaxis(ax1: Axes, ax2: Axes): + y1_lims = ax1.get_ylim() + y2_lims = ax2.get_ylim() + + y1_frac = (0 - y1_lims[0]) / (y1_lims[1] - y1_lims[0]) + y2_frac = (0 - y2_lims[0]) / (y2_lims[1] - y2_lims[0]) + + if y1_frac != y2_frac: + span = y2_lims[1] - y2_lims[0] + new_bottom = y2_lims[0] + (y2_frac - y1_frac) * span + new_top = new_bottom + span + ax2.set_ylim(new_bottom, new_top) + + # Thermal demand + ax.plot(range(len(thermal_demand)), thermal_demand, color='xkcd:soft blue') + ax.fill_between( + range(len(thermal_demand)), + np.zeros_like(thermal_demand), + thermal_demand, + color='xkcd:soft blue', + alpha=0.2 + ) + ax.set_ylabel('Cooling Demand [kWh]') + ax.yaxis.label.set_color('xkcd:soft blue') + + # Energy from battery + ax_twin = ax.twinx() + ax_twin.bar(range(len(energy_from_battery)), energy_from_battery, color='xkcd:orange') + ax_twin.set_ylabel('Thermal Battery (Dis)Charge [kWh]') + ax_twin.yaxis.label.set_color('xkcd:orange') + + # Aligning plots + _align_yaxis(ax, ax_twin) + ax.plot(np.zeros(time_steps), color='black', linestyle='--') + ax.grid('on') + class CostFunctions: """ From 87159f05250c6257a552ce816e8e3dd28c1adf45 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Sun, 28 Jun 2026 16:14:35 +0200 Subject: [PATCH 17/21] Added rendering option to RL evaluation --- scripts/eval.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/eval.py b/scripts/eval.py index f27be4c..f491ab2 100644 --- a/scripts/eval.py +++ b/scripts/eval.py @@ -52,6 +52,7 @@ def parse_arguments(): parser.add_argument('--env_cfgs', type=str, nargs='?', help="Path to the JSON config file") parser.add_argument('--start', type=int, nargs='?', help="Initial simulation time step") parser.add_argument('--end', type=int, nargs='?', help="Ending simulation time step") + parser.add_argument('--render', action='store_true', help="Flag for enabling env rendering") # Wandb logging parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") @@ -254,9 +255,24 @@ def eval(args, seed): if args.env_cfgs is None: with open(f'{args.exp_dir}/env_cfgs/test_cfgs.json', 'r') as f: env_cfgs = json.load(f) + if args.render: + env_cfgs.update({ + 'render_cfgs': { + 'mode': 'on', + 'dir': f'{args.exp_dir}/seed{seed}/render' + } + }) env = LivingLabEnv(**env_cfgs) else: - env = LivingLabEnv.from_json(config=args.env_cfgs) + env = LivingLabEnv.from_json( + config=args.env_cfgs, + update={ + 'render_cfgs': { + 'mode': 'on' if args.render else 'off', + 'dir': f'{args.exp_dir}/seed{seed}/render' + } + } + ) # Modify start and end time step if provided if args.start is not None: From 1a905634d40155141b67a9b11ae5462a2fcf1503 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Mon, 29 Jun 2026 06:58:09 +0200 Subject: [PATCH 18/21] Added predicted electricity prices observations --- config/citylearn_challenge_2023_Building1.json | 15 +++++++++++++++ config/citylearn_challenge_2023_Building2.json | 9 +++++++++ config/citylearn_challenge_2023_Building3.json | 9 +++++++++ livinglab/envs/livinglab_env.py | 12 +++++++++++- livinglab/utils/data_loader.py | 4 ++++ 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/config/citylearn_challenge_2023_Building1.json b/config/citylearn_challenge_2023_Building1.json index 6515a71..3af6f55 100644 --- a/config/citylearn_challenge_2023_Building1.json +++ b/config/citylearn_challenge_2023_Building1.json @@ -67,6 +67,15 @@ "active": false }, "electricity_pricing": { + "active": true + }, + "electricity_pricing_predicted_1": { + "active": true + }, + "electricity_pricing_predicted_2": { + "active": false + }, + "electricity_pricing_predicted_3": { "active": false }, "carbon_intensity": { @@ -75,6 +84,12 @@ "thermal_battery_soc": { "active": true }, + "energy_from_battery": { + "active": true + }, + "energy_from_heat_pump": { + "active": true + }, "net_electricity_consumption": { "active": false }, diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json index fbed0ba..2120e22 100644 --- a/config/citylearn_challenge_2023_Building2.json +++ b/config/citylearn_challenge_2023_Building2.json @@ -69,6 +69,15 @@ "electricity_pricing": { "active": false }, + "electricity_pricing_predicted_1": { + "active": false + }, + "electricity_pricing_predicted_2": { + "active": false + }, + "electricity_pricing_predicted_3": { + "active": false + }, "carbon_intensity": { "active": false }, diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json index 488b65b..c8b12ed 100644 --- a/config/citylearn_challenge_2023_Building3.json +++ b/config/citylearn_challenge_2023_Building3.json @@ -69,6 +69,15 @@ "electricity_pricing": { "active": false }, + "electricity_pricing_predicted_1": { + "active": false + }, + "electricity_pricing_predicted_2": { + "active": false + }, + "electricity_pricing_predicted_3": { + "active": false + }, "carbon_intensity": { "active": false }, diff --git a/livinglab/envs/livinglab_env.py b/livinglab/envs/livinglab_env.py index cac9b15..a49b02c 100644 --- a/livinglab/envs/livinglab_env.py +++ b/livinglab/envs/livinglab_env.py @@ -209,7 +209,7 @@ def info(self) -> Mapping[str, Any]: def observation_names(self) -> List[str]: """Names of all observations that can be returned by the environment.""" sim_data_names = self.energy_simulation.observation_names + self.weather.observation_names + self.pricing.observation_names + self.carbon_intensity.observation_names - device_obs_names = ['thermal_battery_soc', 'net_electricity_consumption', 'underground_temperature'] + device_obs_names = ['thermal_battery_soc', 'energy_from_battery', 'energy_from_heat_pump', 'net_electricity_consumption', 'underground_temperature'] return sim_data_names + device_obs_names @@ -786,6 +786,14 @@ def estimate_observation_space_limits(self, periodic_normalization: Optional[boo low[key] = 0.0 high[key] = 1.0 + elif key == 'energy_from_battery': + low[key] = -self.thermal_battery.capacity + high[key] = self.thermal_battery.capacity + + elif key == 'energy_from_heat_pump': + low[key] = 0.0 + high[key] = self.heat_pump.nominal_power + elif key == 'net_electricity_consumption': low[key] = -self.pv_system.get_generation(inverter_ac_power_per_kw=sim_data['solar_generation'].max()) high[key] = sim_data['non_shiftable_load'].max() + self.heat_pump.nominal_power @@ -1039,6 +1047,8 @@ def _get_observations_data(self, past: bool=True) -> Mapping[str, Union[int, flo observations.update({ 'cooling_demand': self.energy_simulation.cooling_demand[past_t], 'thermal_battery_soc': self.thermal_battery.soc[ep_past_t], + 'energy_from_battery': self.thermal_battery.energy_balance[ep_past_t], + 'energy_from_heat_pump': self.heat_pump.electricity_consumption[ep_past_t], 'net_electricity_consumption': self.net_electricity_consumption[ep_past_t] }) diff --git a/livinglab/utils/data_loader.py b/livinglab/utils/data_loader.py index 2d8a19c..3298159 100644 --- a/livinglab/utils/data_loader.py +++ b/livinglab/utils/data_loader.py @@ -187,6 +187,10 @@ def _load(self, df: pd.DataFrame): # Current electricity pricing self.electricity_pricing = np.array(sim_data['electricity_pricing'], dtype=np.float32) + # Predictions + self.electricity_pricing_predicted_1 = np.array(sim_data['electricity_pricing_predicted_1'], dtype=np.float32) + self.electricity_pricing_predicted_2 = np.array(sim_data['electricity_pricing_predicted_2'], dtype=np.float32) + self.electricity_pricing_predicted_3 = np.array(sim_data['electricity_pricing_predicted_3'], dtype=np.float32) class CarbonEmissions(TimeSeriesData): From ebbdefeb4a31f99613646bc56d2179bd1d287c4b Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Mon, 29 Jun 2026 06:59:00 +0200 Subject: [PATCH 19/21] Added electricity price based rewards --- livinglab/utils/rewards.py | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/livinglab/utils/rewards.py b/livinglab/utils/rewards.py index b25084d..acf0121 100644 --- a/livinglab/utils/rewards.py +++ b/livinglab/utils/rewards.py @@ -124,6 +124,30 @@ def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: reward = -max(0.0, e) return reward + + +class ElectricityCostRewardFunction(RewardFunction): + """ + Electicity import monetary cost reward. + + This reward is designed to penalize high costs due to electricity consumption. + + Parameters + ---------- + :param env_metadata: Static information about the environment. + :type env_metadata: Mapping[str, Any] + """ + def __init__(self, env_metadata: Mapping[str, Any]): + super().__init__(env_metadata) + + def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: + + # Electricity consumption + e = observations['net_electricity_consumption'] + p = observations['electricity_pricing'] + + reward = -max(0.0, e*p*100.0) + return reward class NetElectricityConsumptionAndComfortRewardFunction(RewardFunction): @@ -167,6 +191,58 @@ def coefficients(self, coefficients: Tuple): assert len(coefficients) == len(self.__functions), f'{type(self).__name__} needs {len(self.__functions)} coefficients.' self.__coefficients = coefficients + def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: + # Compute each reward + reward = np.array([f.calculate(observations) for f in self.__functions], dtype=np.float32) + + # Scale rewards by coefficients and sum + reward = reward*self.coefficients + reward = reward.sum(dtype=np.float32).item() + + return reward + + +class ElectricityCostAndComfortRewardFunction(RewardFunction): + """ + Addition of `ElectricityCostRewardFunction` and `ComfortReward`. + + Parameters + ---------- + :param env_metadata: Static information about the environment. + :type env_metadata: Mapping[str, Any] + :param env_metadata: Static information about the environment. + :type env_metadata: Mapping[str, Any] + :param exponent: Exponent to raise the temperature difference to if exceeding `comfort_band`. + :type exponent: Optional[float] + :param coefficients: Coefficents for `NetElectricityConsumption` and `ComfortReward` values respectively. + :type coefficients: Tuple[float], default (1.0, 1.0) + """ + + def __init__(self, env_metadata: Mapping[str, Any], comfort_band: Optional[float]=None, exponent: Optional[float]=None, coefficients: Optional[Tuple[float]]=None): + self.__functions: List[RewardFunction] = [ + ElectricityCostRewardFunction(env_metadata=env_metadata), + ComfortRewardFunction(env_metadata=env_metadata, comfort_band=comfort_band, exponent=exponent) + ] + super().__init__(env_metadata) + self.coefficients = coefficients + + @property + def coefficients(self) -> Tuple: + return self.__coefficients + + @RewardFunction.env_metadata.setter + def env_metadata(self, env_metadata: Mapping[str, Any]) -> Mapping[str, Any]: + RewardFunction.env_metadata.fset(self, env_metadata) + + for f in self.__functions: + f.env_metadata = self.env_metadata + + @coefficients.setter + def coefficients(self, coefficients: Tuple): + coefficients = [1.0]*len(self.__functions) if coefficients is None else coefficients + assert len(coefficients) == len(self.__functions), f'{type(self).__name__} needs {len(self.__functions)} coefficients.' + self.__coefficients = coefficients + def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: # Compute each reward reward = np.array([f.calculate(observations) for f in self.__functions], dtype=np.float32) From e0830c0f51e88a78809a5706a215491a33955f9e Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Mon, 29 Jun 2026 10:54:56 +0200 Subject: [PATCH 20/21] Changes in JSON configurations --- config/citylearn_challenge_2023_Building2.json | 10 ++++++++-- config/citylearn_challenge_2023_Building3.json | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json index 2120e22..71bdcb6 100644 --- a/config/citylearn_challenge_2023_Building2.json +++ b/config/citylearn_challenge_2023_Building2.json @@ -67,10 +67,10 @@ "active": false }, "electricity_pricing": { - "active": false + "active": true }, "electricity_pricing_predicted_1": { - "active": false + "active": true }, "electricity_pricing_predicted_2": { "active": false @@ -84,6 +84,12 @@ "thermal_battery_soc": { "active": true }, + "energy_from_battery": { + "active": true + }, + "energy_from_heat_pump": { + "active": true + }, "net_electricity_consumption": { "active": false }, diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json index c8b12ed..a7ac53f 100644 --- a/config/citylearn_challenge_2023_Building3.json +++ b/config/citylearn_challenge_2023_Building3.json @@ -67,10 +67,10 @@ "active": false }, "electricity_pricing": { - "active": false + "active": true }, "electricity_pricing_predicted_1": { - "active": false + "active": true }, "electricity_pricing_predicted_2": { "active": false @@ -84,6 +84,12 @@ "thermal_battery_soc": { "active": true }, + "energy_from_battery": { + "active": true + }, + "energy_from_heat_pump": { + "active": true + }, "net_electricity_consumption": { "active": false }, From 9ba210cf7661d8dfd6a78903cf01181fb1408426 Mon Sep 17 00:00:00 2001 From: Tr3itz Date: Tue, 30 Jun 2026 15:21:32 +0200 Subject: [PATCH 21/21] Modified rendered GIF --- livinglab/utils/functions.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/livinglab/utils/functions.py b/livinglab/utils/functions.py index 87e2fb8..7660462 100644 --- a/livinglab/utils/functions.py +++ b/livinglab/utils/functions.py @@ -124,14 +124,13 @@ def render_indoor_state( ) # Current indoor/outoor dry-bulb temperature - ax.plot(range(len(indoor_dry_bulb_temperature)), indoor_dry_bulb_temperature, linewidth=2.0, label='Indoor Dry Bulb Temperature', color='orange') - ax.plot(range(len(outdoor_dry_bulb_temperature)), outdoor_dry_bulb_temperature, label='Outdoor Dry Bulb Temperature', color='xkcd:light purple') + ax.plot(range(len(indoor_dry_bulb_temperature)), indoor_dry_bulb_temperature, linewidth=3.0, label='Indoor Dry Bulb Temperature', color='orange') + ax.plot(range(len(outdoor_dry_bulb_temperature)), outdoor_dry_bulb_temperature, linewidth=1.5, label='Outdoor Dry Bulb Temperature', color='xkcd:light purple') # Style ax.grid('on') - ax.set_title('Indoor Dry-bulb Temperature Evolution', fontweight='bold') - ax.set_ylabel('Temperature [°C]') - ax.legend(loc='upper left') + ax.set_ylabel('Temperature [°C]', fontsize=15) + ax.legend(ncols=3, bbox_to_anchor=(0.825, 1.15), fontsize=15) @staticmethod def render_device_control(ax: Axes, thermal_demand: np.ndarray, energy_from_battery: np.ndarray, time_steps: int): @@ -172,13 +171,13 @@ def _align_yaxis(ax1: Axes, ax2: Axes): color='xkcd:soft blue', alpha=0.2 ) - ax.set_ylabel('Cooling Demand [kWh]') + ax.set_ylabel('Cooling Demand [kW]', fontsize=15) ax.yaxis.label.set_color('xkcd:soft blue') # Energy from battery ax_twin = ax.twinx() ax_twin.bar(range(len(energy_from_battery)), energy_from_battery, color='xkcd:orange') - ax_twin.set_ylabel('Thermal Battery (Dis)Charge [kWh]') + ax_twin.set_ylabel('Thermal Battery (Dis)Charge [kW]', fontsize=15) ax_twin.yaxis.label.set_color('xkcd:orange') # Aligning plots