diff --git a/.gitignore b/.gitignore index 3286494..2899912 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,13 @@ # Omnisafe runs **/runs/* +# Installation artifacts +**/*.egg-info/* +**/.vscode/* + +# Bash scripts +**/*.sh + # Experiments and wandb **/tea_debug.log **/wandb/* diff --git a/README.md b/README.md index c24157c..85a8bfb 100644 --- a/README.md +++ b/README.md @@ -50,29 +50,24 @@ We recommend using a [Miniconda](https://www.anaconda.com/docs/getting-started/m ```bash git clone https://github.com/Isla-lab/LivingLab.git ``` -2. Setup [Omnisafe](https://github.com/PKU-Alignment/omnisafe): +2. Create and activate a miniconda environment: ```bash - # 1. Create the conda environment - cd LivingLab/ext/omnisafe - conda env create --file conda-recipe.yaml - - # 2. Install omnisafe - conda activate safe-livinglab + conda create -n livinglab python==3.10 -y + conda activate livinglab + ``` +3. Setup `LivingLab`: + ```bash + cd LivingLab/ pip install -e . ``` -3. Update and install additional libraries +4. Setup [Omnisafe](https://github.com/PKU-Alignment/omnisafe): ```bash - # 1. Update torch and torchvision - pip install torch==2.8.0 torchvision==0.23.0 - - # 2. Install utilities - pip install ipywidgets - pip install ladybug-core - pip install stable_baselines3==2.0.0 + cd ext/omnisafe + pip install -e . ``` ## 2. 🚀 Quick Start -You can easily use the environment by importing it and and calling calling `.gym.make()`. +You can easily use the environment by importing it and and calling calling `gym.make()`. ```python import gymnasium as gym import livinglab # Registers the environment with default configurations diff --git a/config/device_only_obs.json b/config/citylearn_challenge_2023_Building1.json similarity index 87% rename from config/device_only_obs.json rename to config/citylearn_challenge_2023_Building1.json index 00a1f7f..3af6f55 100644 --- a/config/device_only_obs.json +++ b/config/citylearn_challenge_2023_Building1.json @@ -43,7 +43,7 @@ "active": true }, "comfort_band": { - "active": false + "active": true }, "cooling_demand": { "active": false @@ -67,6 +67,15 @@ "active": false }, "electricity_pricing": { + "active": true + }, + "electricity_pricing_predicted_1": { + "active": true + }, + "electricity_pricing_predicted_2": { + "active": false + }, + "electricity_pricing_predicted_3": { "active": false }, "carbon_intensity": { @@ -75,6 +84,12 @@ "thermal_battery_soc": { "active": true }, + "energy_from_battery": { + "active": true + }, + "energy_from_heat_pump": { + "active": true + }, "net_electricity_consumption": { "active": false }, @@ -149,9 +164,16 @@ 0.9749279, 1.0, 43.536755 - ] + ], + "reset_on_ep_start": true }, "periodic_normalization": true, + "thermal_demand_propagation": 1, + "random_ep_reset": false, + "render_cfgs": { + "mode": "off", + "dir": "../render" + }, "reward_fn": { "class": "ComfortRewardFunction" } diff --git a/config/citylearn_challenge_2023_Building2.json b/config/citylearn_challenge_2023_Building2.json new file mode 100644 index 0000000..71bdcb6 --- /dev/null +++ b/config/citylearn_challenge_2023_Building2.json @@ -0,0 +1,180 @@ +{ + "seed": 42, + "sim_data_paths": { + "energy_simulation": "/datasets/citylearn_challenge_2023_phase_1/Building_2.csv", + "weather": "/datasets/citylearn_challenge_2023_phase_1/weather.csv", + "pricing": "/datasets/citylearn_challenge_2023_phase_1/pricing.csv", + "carbon_intensity": "/datasets/citylearn_challenge_2023_phase_1/carbon_intensity.csv" + }, + "start_time_step": 0, + "end_time_step": 719, + "observations_metadata": { + "hour": { + "active": true, + "periodic_metadata": { + "min": 1, + "max": 24 + } + }, + "day_type": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 7 + } + }, + "month": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 12 + } + }, + "indoor_dry_bulb_temperature": { + "active": true + }, + "indoor_relative_humidity": { + "active": false + }, + "occupant_count": { + "active": false + }, + "indoor_dry_bulb_temperature_cooling_set_point": { + "active": true + }, + "comfort_band": { + "active": true + }, + "cooling_demand": { + "active": false + }, + "non_shiftable_load": { + "active": false + }, + "solar_generation": { + "active": false + }, + "outdoor_dry_bulb_temperature": { + "active": true + }, + "outdoor_relative_humidity": { + "active": false + }, + "diffuse_solar_irradiance": { + "active": false + }, + "direct_solar_irradiance": { + "active": false + }, + "electricity_pricing": { + "active": true + }, + "electricity_pricing_predicted_1": { + "active": true + }, + "electricity_pricing_predicted_2": { + "active": false + }, + "electricity_pricing_predicted_3": { + "active": false + }, + "carbon_intensity": { + "active": false + }, + "thermal_battery_soc": { + "active": true + }, + "energy_from_battery": { + "active": true + }, + "energy_from_heat_pump": { + "active": true + }, + "net_electricity_consumption": { + "active": false + }, + "underground_temperature": { + "active": true + } + }, + "heat_pump_cfgs": { + "nominal_power": 2.25, + "efficiency": 0.29, + "mode": "cooling", + "target_temperature": 6.0, + "tank_depth": 5.0, + "soil_alpha": 0.052, + "kasuda_data": "../data/datasets/citylearn_challenge_2023_phase_1/USA_TX_San.Antonio.Intl.AP.722530_TMY3.epw" + }, + "thermal_battery_cfgs": { + "capacity": 4.0, + "efficiency": 0.95, + "loss_coef": 1e-05 + }, + "pv_system_cfgs": { + "nominal_power": 1.2 + }, + "dynamics_cfgs": { + "path": "/datasets/citylearn_challenge_2023_phase_1/Building_2.pth", + "num_layers": 2, + "input_size": 13, + "hidden_size": 16, + "lookback": 12, + "input_observation_names": [ + "direct_solar_irradiance", + "diffuse_solar_irradiance", + "outdoor_dry_bulb_temperature", + "indoor_dry_bulb_temperature_cooling_set_point", + "occupant_count", + "cooling_demand", + "month_sin", + "month_cos", + "hour_sin", + "hour_cos", + "day_type_sin", + "day_type_cos", + "indoor_dry_bulb_temperature" + ], + "input_norm_min": [ + 0.0, + 0.0, + 21.7, + 20.444445, + 0.0, + 0.0, + -0.8660254, + -1.0, + -1.0, + -1.0, + -0.9749279, + -0.90096885, + 16.119997 + ], + "input_norm_max": [ + 931.0, + 486.5, + 42.8, + 26.666666, + 1.0, + 11.077426, + 1.2246469000000002e-16, + -0.5, + 1.0, + 1.0, + 0.9749279, + 1.0, + 42.521355 + ], + "reset_on_ep_start": true + }, + "periodic_normalization": true, + "thermal_demand_propagation": 1, + "random_ep_reset": false, + "render_cfgs": { + "mode": "off", + "dir": "./render" + }, + "reward_fn": { + "class": "ComfortRewardFunction" + } +} \ No newline at end of file diff --git a/config/citylearn_challenge_2023_Building3.json b/config/citylearn_challenge_2023_Building3.json new file mode 100644 index 0000000..a7ac53f --- /dev/null +++ b/config/citylearn_challenge_2023_Building3.json @@ -0,0 +1,180 @@ +{ + "seed": 42, + "sim_data_paths": { + "energy_simulation": "/datasets/citylearn_challenge_2023_phase_1/Building_3.csv", + "weather": "/datasets/citylearn_challenge_2023_phase_1/weather.csv", + "pricing": "/datasets/citylearn_challenge_2023_phase_1/pricing.csv", + "carbon_intensity": "/datasets/citylearn_challenge_2023_phase_1/carbon_intensity.csv" + }, + "start_time_step": 0, + "end_time_step": 719, + "observations_metadata": { + "hour": { + "active": true, + "periodic_metadata": { + "min": 1, + "max": 24 + } + }, + "day_type": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 7 + } + }, + "month": { + "active": false, + "periodic_metadata": { + "min": 1, + "max": 12 + } + }, + "indoor_dry_bulb_temperature": { + "active": true + }, + "indoor_relative_humidity": { + "active": false + }, + "occupant_count": { + "active": false + }, + "indoor_dry_bulb_temperature_cooling_set_point": { + "active": true + }, + "comfort_band": { + "active": true + }, + "cooling_demand": { + "active": false + }, + "non_shiftable_load": { + "active": false + }, + "solar_generation": { + "active": false + }, + "outdoor_dry_bulb_temperature": { + "active": true + }, + "outdoor_relative_humidity": { + "active": false + }, + "diffuse_solar_irradiance": { + "active": false + }, + "direct_solar_irradiance": { + "active": false + }, + "electricity_pricing": { + "active": true + }, + "electricity_pricing_predicted_1": { + "active": true + }, + "electricity_pricing_predicted_2": { + "active": false + }, + "electricity_pricing_predicted_3": { + "active": false + }, + "carbon_intensity": { + "active": false + }, + "thermal_battery_soc": { + "active": true + }, + "energy_from_battery": { + "active": true + }, + "energy_from_heat_pump": { + "active": true + }, + "net_electricity_consumption": { + "active": false + }, + "underground_temperature": { + "active": true + } + }, + "heat_pump_cfgs": { + "nominal_power": 2.78, + "efficiency": 0.26, + "mode": "cooling", + "target_temperature": 6.21, + "tank_depth": 7.5, + "soil_alpha": 0.052, + "kasuda_data": "../data/datasets/citylearn_challenge_2023_phase_1/USA_TX_San.Antonio.Intl.AP.722530_TMY3.epw" + }, + "thermal_battery_cfgs": { + "capacity": 3.3, + "efficiency": 0.96, + "loss_coef": 1e-05 + }, + "pv_system_cfgs": { + "nominal_power": 2.4 + }, + "dynamics_cfgs": { + "path": "/datasets/citylearn_challenge_2023_phase_1/Building_3.pth", + "num_layers": 2, + "input_size": 13, + "hidden_size": 16, + "lookback": 12, + "input_observation_names": [ + "direct_solar_irradiance", + "diffuse_solar_irradiance", + "outdoor_dry_bulb_temperature", + "indoor_dry_bulb_temperature_cooling_set_point", + "occupant_count", + "cooling_demand", + "month_sin", + "month_cos", + "hour_sin", + "hour_cos", + "day_type_sin", + "day_type_cos", + "indoor_dry_bulb_temperature" + ], + "input_norm_min": [ + 0.0, + 0.0, + 21.7, + 21.666666, + 0.0, + 0.0, + -0.8660254, + -1.0, + -1.0, + -1.0, + -0.9749279, + -0.90096885, + 17.278326 + ], + "input_norm_max": [ + 931.0, + 486.5, + 42.8, + 29.444445, + 2.0, + 11.438825, + 1.2246469000000002e-16, + -0.5, + 1.0, + 1.0, + 0.9749279, + 1.0, + 43.541588 + ], + "reset_on_ep_start": true + }, + "periodic_normalization": true, + "thermal_demand_propagation": 1, + "random_ep_reset": false, + "render_cfgs": { + "mode": "off", + "dir": "./render" + }, + "reward_fn": { + "class": "ComfortRewardFunction" + } +} \ No newline at end of file diff --git a/examples/init_env.py b/examples/init_env.py index c7d6e27..1a74407 100644 --- a/examples/init_env.py +++ b/examples/init_env.py @@ -1,8 +1,4 @@ -import sys -import os import json -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - import numpy as np from collections import defaultdict diff --git a/examples/train_omnisafe.py b/examples/train_omnisafe.py index e863d33..e0ad284 100644 --- a/examples/train_omnisafe.py +++ b/examples/train_omnisafe.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # Omnisafe from ext import omnisafe @@ -40,7 +37,7 @@ def main(args): 'seed': args.seed, 'train_cfgs': { 'total_steps': args.episodes*(env_cfgs['episode_length']-1), - 'device': 'cuda' if torch.cuda.is_available() else 'cpu' + 'device': 'cuda:0' if torch.cuda.is_available() else 'cpu' }, 'algo_cfgs': { 'steps_per_epoch': env_cfgs['episode_length']-1, @@ -58,18 +55,12 @@ def main(args): # --- LIVINGLAB KEY ARGUMENTS --- 'env_cfgs': { - **env_cfgs, - 'cost_fn': { - 'name': 'electricity_consumption', - 'kwargs': { - 'exponent': 2.0 - } - } + **env_cfgs } } # 3. Define and train the agent - agent = omnisafe.Agent('PPOLag', 'LivingLab-v0', custom_cfgs=custom_cfgs) + agent = omnisafe.Agent('PPO', 'LivingLab-v0', custom_cfgs=custom_cfgs) agent.learn() diff --git a/examples/train_sb3.py b/examples/train_sb3.py index 963e2c4..4868f56 100644 --- a/examples/train_sb3.py +++ b/examples/train_sb3.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # SB3 from stable_baselines3 import PPO from stable_baselines3.common.callbacks import BaseCallback diff --git a/livinglab/base.py b/livinglab/base.py index 15bbcbc..3f6e84d 100644 --- a/livinglab/base.py +++ b/livinglab/base.py @@ -32,6 +32,7 @@ def __init__(self, seed: Optional[int]=None, start_time_step: Optional[int]=None # Episodic info self.episode_length = episode_length + self._ready_to_reset = True @property def seed(self) -> int: @@ -58,6 +59,11 @@ def episode_length(self): """Time steps duration of a simulation episode.""" return self._episode_length + @property + def ready_to_reset(self) -> bool: + """Wheather the environment can safely call `Environment.reset()`""" + return self._ready_to_reset + @property def simulation_episodes(self): """Number of simulation episodes.""" @@ -101,6 +107,8 @@ def start_time_step(self, new_step: Optional[int]): f'Invalid simulation start/end steps (start={new_step} >= end={self._end_time_step}).' self._start_time_step = new_step + if hasattr(self, '_episode_length'): + self.episode_length = min(self._episode_length, (self._end_time_step - self._start_time_step) + 1) @end_time_step.setter def end_time_step(self, new_step: Optional[int]): @@ -110,6 +118,8 @@ def end_time_step(self, new_step: Optional[int]): f'Invalid simulation start/end steps (start={self._start_time_step} >= end={new_step}).' self._end_time_step = new_step + if hasattr(self, '_episode_length'): + self.episode_length = min(self._episode_length, (self._end_time_step - self._start_time_step) + 1) @episode_length.setter def episode_length(self, new_len: Optional[int]): @@ -130,6 +140,10 @@ def episode_length(self, new_len: Optional[int]): self.simulation_episodes = (self._end_time_step + 1) // self.episode_length self.episode_counter = -1 + @ready_to_reset.setter + def ready_to_reset(self, new_val: bool): + self._ready_to_reset = new_val + @simulation_episodes.setter def simulation_episodes(self, n: int): assert n > 0, f'Invalid number of simulation episodes n={n}. Must be > 0.' @@ -160,16 +174,25 @@ def episode_end_time_step(self, new_step: int): def step(self): self.episode_time_step += 1 + self._ready_to_reset = True - def reset(self): + def reset(self, episode_start_time_step: Optional[int]=None): """ Reset the environment to its initial state, and set next `self.episode_start_time_step` and `self.episode_end_time_step` correclty. + + Parameters + ---------- + :param episode_start_time_step: Explicit time step to reset the environment to + :type episode_start_time_step: int """ - self.episode_counter += 1 - self.episode_time_step = 0 - self.episode_start_time_step = self.start_time_step + (self.episode_counter % self.simulation_episodes) * self.episode_length - self.episode_end_time_step = self.episode_start_time_step + self.episode_length - 1 + if self.ready_to_reset: + self.episode_counter += 1 + self.episode_time_step = 0 + self.episode_start_time_step = episode_start_time_step if episode_start_time_step is not None else \ + self.start_time_step + (self.episode_counter % self.simulation_episodes) * self.episode_length + self.episode_end_time_step = self.episode_start_time_step + self.episode_length - 1 + self._ready_to_reset = False class Device(Environment): diff --git a/livinglab/components/battery.py b/livinglab/components/battery.py index 22bde43..a258ad6 100644 --- a/livinglab/components/battery.py +++ b/livinglab/components/battery.py @@ -155,9 +155,16 @@ def charge(self, energy: float): else: self._energy_balance[self.episode_time_step] = delta_energy * self.round_trip_efficiency - def reset(self): - """Reset the Thermal Battery to its initial state.""" - super().reset() + def reset(self, **kwargs: Mapping[str, Any]): + """ + Reset the Thermal Battery to its initial state. + + Parameters + ---------- + :param kwargs: Keyword parameters for `super().reset()` + :type kwargs: Mapping[str, Any] + """ + super().reset(**kwargs) self._soc = np.zeros(self.episode_length, dtype=np.float32) self._soc[0] = self.initial_soc self._energy_balance = np.zeros(self.episode_length, dtype=np.float32) diff --git a/livinglab/components/device.py b/livinglab/components/device.py index 79b6771..746eafa 100644 --- a/livinglab/components/device.py +++ b/livinglab/components/device.py @@ -57,9 +57,16 @@ def update_electricity_consumption(self, electricity_consumption: float, enforce f'Invalid electricity consumption value {electricity_consumption}. Must be >= 0.' self._electricity_consumption[self.episode_time_step] += electricity_consumption - def reset(self): - """Reset the Electric Device to its initial state.""" - super().reset() + def reset(self, **kwargs: Mapping[str, Any]): + """ + Reset the Electric Device to its initial state. + + Parameters + ---------- + :param kwargs: Keyword parameters for `super().reset()` + :type kwargs: Mapping[str, Any] + """ + super().reset(**kwargs) self._electricity_consumption = np.zeros(self.episode_length, dtype=np.float32) def get_metadata(self) -> Mapping[str, Any]: diff --git a/livinglab/components/dynamics.py b/livinglab/components/dynamics.py index fb9744f..054404d 100644 --- a/livinglab/components/dynamics.py +++ b/livinglab/components/dynamics.py @@ -3,7 +3,7 @@ from pathlib import Path from abc import ABC, abstractmethod -from typing import Tuple, List, Union +from typing import Optional, Tuple, List, Union class Dynamics(ABC): """ @@ -30,8 +30,9 @@ def __init__( num_layers: int, hidden_size: int, lookback: int, - input_size: int=None, - dropout: float=0.0 + input_size: Optional[int]=None, + dropout: float=0.0, + reset_on_ep_start: Optional[bool]=None ): Dynamics.__init__(self) nn.Module.__init__(self) @@ -61,6 +62,9 @@ def __init__( in_features=self.hidden_size, out_features=1, # <- the predicted indoor temperature ) + + # Dynamics options + self.reset_on_ep_start = reset_on_ep_start @property def model_input(self): @@ -82,6 +86,10 @@ def hidden_state(self): def input_size(self): return self._input_size + @property + def reset_on_ep_start(self) -> bool: + return self._reset_on_ep_start + @input_size.setter def input_size(self, new_size: int): assert new_size is None or new_size > 0, f'Invalid input dimensionality {new_size}. Must be either `None` or > 0.' @@ -95,6 +103,11 @@ def model_input(self, new_input): def hidden_state(self, new_h: Tuple[torch.Tensor, torch.Tensor]): self._hidden_state = new_h + @reset_on_ep_start.setter + def reset_on_ep_start(self, new_val: Optional[bool]): + assert new_val is None or isinstance(new_val, bool), f'Invalid type for `LSTMDynamics.reset_on_ep_start`. Required `bool`, found {type(new_val)}.' + self._reset_on_ep_start = True if new_val is None else new_val + def forward(self, x: torch.Tensor, h: Tuple[torch.Tensor, torch.Tensor]) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: lstm_out, h = self.l_lstm(x, h) lstm_out = self.dropout(lstm_out) diff --git a/livinglab/envs/livinglab_env.py b/livinglab/envs/livinglab_env.py index cfd0cd1..a49b02c 100644 --- a/livinglab/envs/livinglab_env.py +++ b/livinglab/envs/livinglab_env.py @@ -1,10 +1,14 @@ import torch import numpy as np import gymnasium as gym +import imageio.v2 as imageio from gymnasium import spaces +from PIL import Image +from matplotlib import pyplot as plt import os import json +from glob import glob from pathlib import Path from typing import Any, Literal, Optional, Union, Callable, Iterable, Tuple, Set, List, Mapping, Dict from typing_extensions import Self @@ -13,7 +17,7 @@ from livinglab.components.dynamics import Dynamics, LSTMDynamics from livinglab.components.device import DualSourceHeatPump, PVSystem from livinglab.components.battery import ThermalBattery -from livinglab.utils.functions import DAYS_PER_MONTH, CostFunctions +from livinglab.utils.functions import DAYS_PER_MONTH, UtilsFunctions, CostFunctions from livinglab.utils.data_loader import EnergySimulation, Weather, Pricing, CarbonEmissions from livinglab.utils.preprocessing import Normalize, PeriodicNormalization from livinglab.utils import rewards @@ -65,6 +69,9 @@ def __init__( active_observations: Optional[Iterable[str]]=[], inactive_observations: Optional[Iterable[str]]=[], periodic_observations_metadata: Optional[Mapping[str, Iterable[Union[int, float]]]]=None, + thermal_demand_propagation: Optional[int]=None, + random_ep_reset: Optional[bool]=None, + render_cfgs: Optional[Mapping[str, Any]]=None, episode_length: Optional[int]=None, ): super().__init__(seed=seed, start_time_step=start_time_step, end_time_step=end_time_step, episode_length=episode_length) @@ -78,6 +85,10 @@ def __init__( # Options self.periodic_normalization = periodic_normalization self.periodic_observations_metadata = periodic_observations_metadata + self.thermal_demand_propagation = thermal_demand_propagation + self.random_ep_reset = random_ep_reset + self.render_mode = None if render_cfgs is None else render_cfgs.get('mode', 'off') + self.render_dir = None if render_cfgs is None else render_cfgs.get('dir', None) self.active_observations = set(active_observations) self.inactive_observations = set(inactive_observations) assert len(self.active_observations.intersection(inactive_observations)) == 0, \ @@ -198,7 +209,7 @@ def info(self) -> Mapping[str, Any]: def observation_names(self) -> List[str]: """Names of all observations that can be returned by the environment.""" sim_data_names = self.energy_simulation.observation_names + self.weather.observation_names + self.pricing.observation_names + self.carbon_intensity.observation_names - device_obs_names = ['thermal_battery_soc', 'net_electricity_consumption', 'underground_temperature'] + device_obs_names = ['thermal_battery_soc', 'energy_from_battery', 'energy_from_heat_pump', 'net_electricity_consumption', 'underground_temperature'] return sim_data_names + device_obs_names @@ -246,6 +257,40 @@ def action_space(self) -> spaces.Box: """Environment's action space.""" return self._action_space + @property + def thermal_demand_propagation(self) -> int: + """ + Environment's mode of propagating the thermal demand generated at `self.time_step` to the next time step. + * 0 -> thermal demand affects the indoor dry-bulb temperature at `self.time_step` (no propagation) + * 1 -> thermal demand affects the indoor dry-bulb temperature at `self.time_step + 1` + * 2 -> thermal demand affects the indoor dry-bulb temperature at both `self.time_step` and `self.time_step + 1` + """ + return self._thermal_demand_propagation + + @property + def random_ep_reset(self) -> bool: + """Whether to restart each episode to a random initial time step.""" + return self._random_ep_reset + + @property + def render_mode(self) -> str: + """ + Living Lab render mode. + * `off` -> no render + * `on` -> render figures are saved to `self.render_dir` + * `real` -> real-time render + saving figures + """ + return self._render_mode + + @property + def render_dir(self) -> Union[str, None]: + return self._render_dir + + @property + def reset_dynamics(self) -> bool: + """Whether to call `self.dynamics.reset()` upon calling `self.reset()`.""" + return self.episode_counter < 1 or self.dynamics.reset_on_ep_start + @property def env_metadata(self) -> Mapping[str, Any]: return { @@ -325,6 +370,32 @@ def observation_space(self, new_space: spaces.Box): def action_space(self, new_space: spaces.Box): self._action_space = new_space + @thermal_demand_propagation.setter + def thermal_demand_propagation(self, new_mode: Optional[int]): + assert new_mode is None or new_mode in range(3), f'Invalid thermal demand propagation mode {new_mode}. Must be in [0, 1, 2].' + new_mode = 0 if new_mode is None else new_mode + self._thermal_demand_propagation = new_mode + + @random_ep_reset.setter + def random_ep_reset(self, new_val: Optional[bool]): + assert new_val is None or isinstance(new_val, bool), f'Invalid type for `LivingLabEnv.random_ep_reset`. Required `bool`, found {type(new_val)}.' + self._random_ep_reset = False if new_val is None else new_val + + @render_mode.setter + def render_mode(self, new_mode: Optional[str]): + assert new_mode is None or new_mode in ['off', 'on', 'real'], f'Invalid render mode {new_mode}. Must be in [`off`, `on`, `real`].' + + # TODO: work on real-time rendering + if new_mode == 'real': + print('[WARN] Real-time rendering has not been implemented, yet. Switched to render_mode=`on`.') + + self._render_mode = 'off' if new_mode is None else new_mode + + @render_dir.setter + def render_dir(self, new_dir: Optional[str]): + self._render_dir = '../render' if new_dir is None else new_dir + os.makedirs(self._render_dir, exist_ok=True) + def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tuple[np.ndarray, Mapping[str, Any]]: """ Reset `LivingLabEnv` to its initial state. @@ -342,7 +413,15 @@ def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tu :rtype: np.ndarray """ gym.Env.reset(self) - Environment.reset(self) + + # Reset the environment to a valid random start time step + if self.random_ep_reset: + random_start_time_step = np.random.choice( + np.arange(self.start_time_step, self.end_time_step, self.episode_length) + ) + else: + random_start_time_step = None + Environment.reset(self, episode_start_time_step=random_start_time_step) # Check options if options is None: @@ -353,12 +432,13 @@ def reset(self, seed: int=None, options: Optional[Mapping[str, Any]]=None) -> Tu self.seed = seed # Reset devices - self.heat_pump.reset() - self.thermal_battery.reset() - self.pv_system.reset() + self.heat_pump.reset(episode_start_time_step=random_start_time_step) + self.thermal_battery.reset(episode_start_time_step=random_start_time_step) + self.pv_system.reset(episode_start_time_step=random_start_time_step) # Reset dynamics - self.dynamics.reset() + if self.reset_dynamics: + self.dynamics.reset() # Reset additional variables self._episode_rewards = np.zeros(self.episode_length, dtype=np.float32) @@ -398,9 +478,21 @@ def step(self, actions: Union[np.ndarray | List[float]]) -> Tuple[np.ndarray, fl self.apply_actions(**action_dict) # Update indoor temperature via dynamics - self._update_dynamics_input() - if self.simulate_dynamics: - self.update_indoor_dry_bulb_temperature() + if self._thermal_demand_propagation == 0: + self._extend_dynamics_input() + if self.simulate_dynamics: + self.update_indoor_dry_bulb_temperature() + else: + if self.simulate_dynamics: + if self._thermal_demand_propagation == 2: + self._update_last_dynamics_input( + vars={ + 'cooling_demand': self.energy_simulation.cooling_demand[self.time_step] + } + ) + self.update_indoor_dry_bulb_temperature() + else: + self._extend_dynamics_input() # Update environment variables (reflect effects of actions) net_electricity_consumption = ( @@ -421,8 +513,69 @@ def step(self, actions: Union[np.ndarray | List[float]]) -> Tuple[np.ndarray, fl # Advance to the next time step self._next_time_step() + # TODO - ASSUMPTION: propagate the last agent's action to the current state + if self.simulate_dynamics and self._thermal_demand_propagation > 0: + # Extract the last cooling demand (result of the agent's action) + self._extend_dynamics_input(past=True) + + # Update the indoor temperature at the current `self.time_step` using the current observations + # In this way the agent "sees" the propagation of its actions through time + self.update_indoor_dry_bulb_temperature() + + # TODO: render + if self.render_mode != 'off': + self.render() + return self.observations(), reward, self.terminated, self.truncated, self.info + def render(self): + # Retrieve data + indoor_dry_bulb_temperature = self.energy_simulation.indoor_dry_bulb_temperature[self.episode_start_time_step:self.time_step] + indoor_temperature_setpoint = self.energy_simulation.indoor_dry_bulb_temperature_cooling_set_point[self.episode_start_time_step:self.episode_end_time_step+1] + comfort_band = self.energy_simulation.comfort_band[self.episode_start_time_step:self.episode_end_time_step+1] + outdoor_dry_bulb_temperature = self.weather.outdoor_dry_bulb_temperature[self.episode_start_time_step:self.time_step] + cooling_demand = self.energy_simulation.cooling_demand[self.episode_start_time_step:self.time_step] + energy_from_battery = self.thermal_battery.energy_balance[:self.episode_time_step] + + # Create figure + fig = plt.figure(figsize=[20,10]) + + # Indoor dry-bulb temperature + ax = fig.add_subplot(2,1,1) + UtilsFunctions.render_indoor_state( + ax=ax, + indoor_dry_bulb_temperature=indoor_dry_bulb_temperature, + indoor_temperature_setpoint=indoor_temperature_setpoint, + comfort_band=comfort_band, + outdoor_dry_bulb_temperature=outdoor_dry_bulb_temperature, + time_steps=self.episode_length + ) + + # Thermal demand and Battery energy balance + bx = fig.add_subplot(2,1,2) + UtilsFunctions.render_device_control( + ax=bx, + thermal_demand=cooling_demand, + energy_from_battery=energy_from_battery, + time_steps=self.episode_length + ) + + # Save the figure + os.makedirs(f'{self.render_dir}/ep{self.episode_counter:04}', exist_ok=True) + fig.savefig(f'{self.render_dir}/ep{self.episode_counter:04}/step{self.episode_time_step:03}.png', format='png') + plt.close(fig=fig) + + # Create GIF animation + if self.terminated: + image_files = sorted(glob(f'{self.render_dir}/ep{self.episode_counter:04}/*.png')) + imgs = [Image.open(f).convert('P', palette=Image.ADAPTIVE) for f in image_files] + imgs[0].save( + f'{self.render_dir}/ep{self.episode_counter:04}/animation.gif', + append_images=imgs[1:], + duration=100, + loop=0 + ) + def load_device(self, device: Union[Device, Mapping[str, Any]], device_class: Callable) -> Device: """ Load a simulation-ready device. @@ -633,6 +786,14 @@ def estimate_observation_space_limits(self, periodic_normalization: Optional[boo low[key] = 0.0 high[key] = 1.0 + elif key == 'energy_from_battery': + low[key] = -self.thermal_battery.capacity + high[key] = self.thermal_battery.capacity + + elif key == 'energy_from_heat_pump': + low[key] = 0.0 + high[key] = self.heat_pump.nominal_power + elif key == 'net_electricity_consumption': low[key] = -self.pv_system.get_generation(inverter_ac_power_per_kw=sim_data['solar_generation'].max()) high[key] = sim_data['non_shiftable_load'].max() + self.heat_pump.nominal_power @@ -884,8 +1045,10 @@ def _get_observations_data(self, past: bool=True) -> Mapping[str, Union[int, flo past_t = max(self.time_step - 1, 0) if past else self.time_step ep_past_t = max(self.episode_time_step - 1, 0) if past else self.episode_time_step observations.update({ - 'cooling_demand': self.energy_simulation.cooling_demand[past_t] + abs(min(self.thermal_battery.energy_balance[ep_past_t], 0.0)), + 'cooling_demand': self.energy_simulation.cooling_demand[past_t], 'thermal_battery_soc': self.thermal_battery.soc[ep_past_t], + 'energy_from_battery': self.thermal_battery.energy_balance[ep_past_t], + 'energy_from_heat_pump': self.heat_pump.electricity_consumption[ep_past_t], 'net_electricity_consumption': self.net_electricity_consumption[ep_past_t] }) @@ -900,9 +1063,9 @@ def _next_time_step(self): # Increate `self.time_step` Environment.step(self) - def _update_dynamics_input(self, sanity_check: bool=True): + def _extend_dynamics_input(self, sanity_check: bool=True, past: bool=False): # Get current observations - obs = self.observations(include_all=True, past=False, names=True) + obs = self.observations(include_all=True, past=past, names=True) if sanity_check: missing = [name for name in self.dynamics.input_observation_names if name not in obs.keys()] assert len(missing) == 0, f'Missing observations required by the dynamics: {missing}.' @@ -918,6 +1081,21 @@ def _update_dynamics_input(self, sanity_check: bool=True): ) ] + def _update_last_dynamics_input(self, vars: Mapping[str, float], sanity_check: bool=True): + if sanity_check: + missing = [name for name in vars.keys() if name not in self.dynamics.input_observation_names] + assert len(missing) == 0, f'Trying to update variables missing in the dynamics input: {missing}.' + + for name, value in vars.items(): + idx = self.dynamics.input_observation_names.index(name) + + # Normalization values + min_ = self.dynamics.input_norm_min[idx] + max_ = self.dynamics.input_norm_max[idx] + + # Update last value in the dynamics input + self.dynamics.model_input[idx][-1] = (value - min_)/(max_ - min_) + def _get_dynamics_input(self) -> torch.Tensor: model_input = [] diff --git a/livinglab/utils/data_loader.py b/livinglab/utils/data_loader.py index 2d8a19c..3298159 100644 --- a/livinglab/utils/data_loader.py +++ b/livinglab/utils/data_loader.py @@ -187,6 +187,10 @@ def _load(self, df: pd.DataFrame): # Current electricity pricing self.electricity_pricing = np.array(sim_data['electricity_pricing'], dtype=np.float32) + # Predictions + self.electricity_pricing_predicted_1 = np.array(sim_data['electricity_pricing_predicted_1'], dtype=np.float32) + self.electricity_pricing_predicted_2 = np.array(sim_data['electricity_pricing_predicted_2'], dtype=np.float32) + self.electricity_pricing_predicted_3 = np.array(sim_data['electricity_pricing_predicted_3'], dtype=np.float32) class CarbonEmissions(TimeSeriesData): diff --git a/livinglab/utils/functions.py b/livinglab/utils/functions.py index 75e4e84..7660462 100644 --- a/livinglab/utils/functions.py +++ b/livinglab/utils/functions.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +from matplotlib.axes import Axes from ladybug.epw import EPW from typing import Optional, Union, Tuple, List, Mapping @@ -85,6 +86,105 @@ def extract_kasuda_parameters(path: str) -> Mapping[str, Union[int, float]]: return {'mean': t_mean, 'amplitude': amplitude, 't0': min_day} + @staticmethod + def render_indoor_state( + ax: Axes, + indoor_dry_bulb_temperature: np.ndarray, + indoor_temperature_setpoint: np.ndarray, + comfort_band: np.ndarray, + outdoor_dry_bulb_temperature: np.ndarray, + time_steps: int, + ): + """ + Render the current indoor state fo the Living Lab. + + Parameters + ---------- + :param ax: `matplotlib.Axes` + :type: Axes + :param indoor_dry_bulb_temperature: Current history of the indoor dry-bulb temperature + :type indoor_dry_bulb_temperature: np.ndarray + :param indoor_temperature_setpoint: User-defined setpoint throughout simulation. + :type indoor_temperature_setpoint: np.ndarray + :param comfort_band: Maximum deviation from the setpoint in terms of degrees to define comfort. + :type comfort_band: np.ndarray + :param outdoor_dry_bulb_temperature: Current history of the outdoor dry-bulb temperature + :type outdoor_dry_bulb_temperature: np.ndarray + :param time_steps: Simulation length + :type time_steps: int + """ + # Setpoint and comfort band + ax.fill_between( + range(time_steps), + indoor_temperature_setpoint + comfort_band, + indoor_temperature_setpoint - comfort_band, + color='g', + alpha=0.15, + label='Comfort band', + ) + + # Current indoor/outoor dry-bulb temperature + ax.plot(range(len(indoor_dry_bulb_temperature)), indoor_dry_bulb_temperature, linewidth=3.0, label='Indoor Dry Bulb Temperature', color='orange') + ax.plot(range(len(outdoor_dry_bulb_temperature)), outdoor_dry_bulb_temperature, linewidth=1.5, label='Outdoor Dry Bulb Temperature', color='xkcd:light purple') + + # Style + ax.grid('on') + ax.set_ylabel('Temperature [°C]', fontsize=15) + ax.legend(ncols=3, bbox_to_anchor=(0.825, 1.15), fontsize=15) + + @staticmethod + def render_device_control(ax: Axes, thermal_demand: np.ndarray, energy_from_battery: np.ndarray, time_steps: int): + """ + Render the current device control. + + Parameters + ---------- + :param ax: `matplotlib.Axes` + :type: Axes + :param thermal_demand: Current thermal demand history due to MSHP control. + :type thermal_demand: np.ndarray + :param energy_from_battery: Current thermal battery energy balance evolution. + :type energy_from_battery: np.ndarray + :param time_steps: Simulation length + :type time_steps: int + """ + + def _align_yaxis(ax1: Axes, ax2: Axes): + y1_lims = ax1.get_ylim() + y2_lims = ax2.get_ylim() + + y1_frac = (0 - y1_lims[0]) / (y1_lims[1] - y1_lims[0]) + y2_frac = (0 - y2_lims[0]) / (y2_lims[1] - y2_lims[0]) + + if y1_frac != y2_frac: + span = y2_lims[1] - y2_lims[0] + new_bottom = y2_lims[0] + (y2_frac - y1_frac) * span + new_top = new_bottom + span + ax2.set_ylim(new_bottom, new_top) + + # Thermal demand + ax.plot(range(len(thermal_demand)), thermal_demand, color='xkcd:soft blue') + ax.fill_between( + range(len(thermal_demand)), + np.zeros_like(thermal_demand), + thermal_demand, + color='xkcd:soft blue', + alpha=0.2 + ) + ax.set_ylabel('Cooling Demand [kW]', fontsize=15) + ax.yaxis.label.set_color('xkcd:soft blue') + + # Energy from battery + ax_twin = ax.twinx() + ax_twin.bar(range(len(energy_from_battery)), energy_from_battery, color='xkcd:orange') + ax_twin.set_ylabel('Thermal Battery (Dis)Charge [kW]', fontsize=15) + ax_twin.yaxis.label.set_color('xkcd:orange') + + # Aligning plots + _align_yaxis(ax, ax_twin) + ax.plot(np.zeros(time_steps), color='black', linestyle='--') + ax.grid('on') + class CostFunctions: """ diff --git a/livinglab/utils/rewards.py b/livinglab/utils/rewards.py index b25084d..acf0121 100644 --- a/livinglab/utils/rewards.py +++ b/livinglab/utils/rewards.py @@ -124,6 +124,30 @@ def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: reward = -max(0.0, e) return reward + + +class ElectricityCostRewardFunction(RewardFunction): + """ + Electicity import monetary cost reward. + + This reward is designed to penalize high costs due to electricity consumption. + + Parameters + ---------- + :param env_metadata: Static information about the environment. + :type env_metadata: Mapping[str, Any] + """ + def __init__(self, env_metadata: Mapping[str, Any]): + super().__init__(env_metadata) + + def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: + + # Electricity consumption + e = observations['net_electricity_consumption'] + p = observations['electricity_pricing'] + + reward = -max(0.0, e*p*100.0) + return reward class NetElectricityConsumptionAndComfortRewardFunction(RewardFunction): @@ -167,6 +191,58 @@ def coefficients(self, coefficients: Tuple): assert len(coefficients) == len(self.__functions), f'{type(self).__name__} needs {len(self.__functions)} coefficients.' self.__coefficients = coefficients + def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: + # Compute each reward + reward = np.array([f.calculate(observations) for f in self.__functions], dtype=np.float32) + + # Scale rewards by coefficients and sum + reward = reward*self.coefficients + reward = reward.sum(dtype=np.float32).item() + + return reward + + +class ElectricityCostAndComfortRewardFunction(RewardFunction): + """ + Addition of `ElectricityCostRewardFunction` and `ComfortReward`. + + Parameters + ---------- + :param env_metadata: Static information about the environment. + :type env_metadata: Mapping[str, Any] + :param env_metadata: Static information about the environment. + :type env_metadata: Mapping[str, Any] + :param exponent: Exponent to raise the temperature difference to if exceeding `comfort_band`. + :type exponent: Optional[float] + :param coefficients: Coefficents for `NetElectricityConsumption` and `ComfortReward` values respectively. + :type coefficients: Tuple[float], default (1.0, 1.0) + """ + + def __init__(self, env_metadata: Mapping[str, Any], comfort_band: Optional[float]=None, exponent: Optional[float]=None, coefficients: Optional[Tuple[float]]=None): + self.__functions: List[RewardFunction] = [ + ElectricityCostRewardFunction(env_metadata=env_metadata), + ComfortRewardFunction(env_metadata=env_metadata, comfort_band=comfort_band, exponent=exponent) + ] + super().__init__(env_metadata) + self.coefficients = coefficients + + @property + def coefficients(self) -> Tuple: + return self.__coefficients + + @RewardFunction.env_metadata.setter + def env_metadata(self, env_metadata: Mapping[str, Any]) -> Mapping[str, Any]: + RewardFunction.env_metadata.fset(self, env_metadata) + + for f in self.__functions: + f.env_metadata = self.env_metadata + + @coefficients.setter + def coefficients(self, coefficients: Tuple): + coefficients = [1.0]*len(self.__functions) if coefficients is None else coefficients + assert len(coefficients) == len(self.__functions), f'{type(self).__name__} needs {len(self.__functions)} coefficients.' + self.__coefficients = coefficients + def calculate(self, observations: Mapping[str, Union[int, float]]) -> float: # Compute each reward reward = np.array([f.calculate(observations) for f in self.__functions], dtype=np.float32) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4a84ecc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "livinglab" +version = "0.1.0" +description = "LivingLab: Intelligent Energy Management RL Environment" +readme = "README.md" +requires-python = ">=3.8" +authors = [{ name = "ISLa Lab" }] +dependencies = [ + "gymnasium>=0.28.1", + "numpy", + "pandas", + "ladybug-core", + "tabulate", + "ipywidgets", + "torch==2.8.0", # + "torchvision==0.23.0", # -> compatability with Omnisafe + "stable-baselines3==2.0.0", # + # Add any other required packages here +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["livinglab*", "ext*"] + +[tool.setuptools.package-data] +livinglab = ["*"] \ No newline at end of file diff --git a/scripts/agent.py b/scripts/agent.py index 2f9a5c2..79c0a1f 100644 --- a/scripts/agent.py +++ b/scripts/agent.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # LivingLab from livinglab.envs.livinglab_env import LivingLabEnv @@ -140,11 +137,11 @@ def predict(self, observations: Mapping[str, Any]) -> np.ndarray: # Above the set-point if hot_delta > 0: if hot_delta > self.comfort_band: # <- Too hot - scheduled_actions[hp_idx] = 0.8*np.sign(scheduled_actions[hp_idx]) + scheduled_actions[hp_idx] = 0.35*np.sign(scheduled_actions[hp_idx]) if thm_soc > 0.1: scheduled_actions[thm_idx] = min(scheduled_actions[thm_idx], -thm_soc/2) else: # <- Hot within the band - scheduled_actions[hp_idx] = 0.2*np.sign(scheduled_actions[hp_idx]) + scheduled_actions[hp_idx] = 0.3*np.sign(scheduled_actions[hp_idx]) if thm_soc > 0.1: scheduled_actions[thm_idx] = min(scheduled_actions[thm_idx], -thm_soc/3) @@ -152,7 +149,7 @@ def predict(self, observations: Mapping[str, Any]) -> np.ndarray: else: temp_delta = outdoor_dry_bulb_temperature - indoor_dry_bulb_temperature if temp_delta > 0: # Outdoor temperature affects indoors - scheduled_actions[hp_idx] = 0.3*np.sign(scheduled_actions[hp_idx]) + scheduled_actions[hp_idx] = 0.15*np.sign(scheduled_actions[hp_idx]) else: scheduled_actions[hp_idx] = 0.0 diff --git a/scripts/eval.py b/scripts/eval.py index 1264d6a..f491ab2 100644 --- a/scripts/eval.py +++ b/scripts/eval.py @@ -1,11 +1,9 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # LivingLab from livinglab.envs.livinglab_env import LivingLabEnv from livinglab.utils.wrappers import NormalizedSpaceWrapper # Utils +import os import wandb import json, yaml import argparse @@ -51,6 +49,10 @@ def parse_arguments(): # LivingLab configs parser.add_argument('--exp_dir', type=str, default='./experiments/PPOLag_net_consumption_reward_discomfort_cost', help="Path to the experiment to evaluate") + parser.add_argument('--env_cfgs', type=str, nargs='?', help="Path to the JSON config file") + parser.add_argument('--start', type=int, nargs='?', help="Initial simulation time step") + parser.add_argument('--end', type=int, nargs='?', help="Ending simulation time step") + parser.add_argument('--render', action='store_true', help="Flag for enabling env rendering") # Wandb logging parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") @@ -126,9 +128,9 @@ def compare_temperature(args, results): bx1.plot(np.zeros_like(avg_demand), color='black', linestyle='--') bx1.grid('on') - os.makedirs(f'{args.exp_dir}/figs', exist_ok=True) - fig.savefig(f'{args.exp_dir}/figs/indoor_dry_bulb_temperature.png', format='png') - plt.show() + env_name = 'test' if args.env_cfgs is None else args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'{args.exp_dir}/figs/{env_name}', exist_ok=True) + fig.savefig(f'{args.exp_dir}/figs/{env_name}/indoor_dry_bulb_temperature.png', format='png') def compare_hp_usage(args, results): @@ -185,9 +187,9 @@ def compare_hp_usage(args, results): bx.legend() bx.grid('on') - os.makedirs(f'{args.exp_dir}/figs', exist_ok=True) - fig.savefig(f'{args.exp_dir}/figs/mshp_usage.png', format='png') - plt.show() + env_name = 'test' if args.env_cfgs is None else args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'{args.exp_dir}/figs/{env_name}', exist_ok=True) + fig.savefig(f'{args.exp_dir}/figs/{env_name}/mshp_usage.png', format='png') def compare_thermal_battery(args, results): @@ -210,10 +212,10 @@ def compare_thermal_battery(args, results): bx2.set_ylabel('SoC [%]') bx2.set_ylim(ymin=-0.05, ymax=1.05) bx2.yaxis.label.set_color('xkcd:orange') - - os.makedirs(f'{args.exp_dir}/figs', exist_ok=True) - fig.savefig(f'{args.exp_dir}/figs/thermal_battery.png', format='png') - plt.show() + + env_name = 'test' if args.env_cfgs is None else args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'{args.exp_dir}/figs/{env_name}', exist_ok=True) + fig.savefig(f'{args.exp_dir}/figs/{env_name}/thermal_battery.png', format='png') def log_kpis(args, results): @@ -248,9 +250,36 @@ def log_kpis(args, results): run.finish() -def eval(args, env_cfgs, seed): +def eval(args, seed): # Load the LivingLabEnv given the configurations - env = LivingLabEnv(**env_cfgs) + if args.env_cfgs is None: + with open(f'{args.exp_dir}/env_cfgs/test_cfgs.json', 'r') as f: + env_cfgs = json.load(f) + if args.render: + env_cfgs.update({ + 'render_cfgs': { + 'mode': 'on', + 'dir': f'{args.exp_dir}/seed{seed}/render' + } + }) + env = LivingLabEnv(**env_cfgs) + else: + env = LivingLabEnv.from_json( + config=args.env_cfgs, + update={ + 'render_cfgs': { + 'mode': 'on' if args.render else 'off', + 'dir': f'{args.exp_dir}/seed{seed}/render' + } + } + ) + + # Modify start and end time step if provided + if args.start is not None: + env.start_time_step = args.start + if args.end is not None: + env.end_time_step = args.end + env = NormalizedSpaceWrapper(env) # Load the agent @@ -308,13 +337,9 @@ def eval(args, env_cfgs, seed): if __name__ == '__main__': args = parse_arguments() - # Load the test configurations - with open(f'{args.exp_dir}/env_cfgs/test_cfgs.json', 'r') as f: - env_cfgs = json.load(f) - results = defaultdict(dict) for i in range(1, len(glob(f'{args.exp_dir}/seed*'))+1): - results[f'seed{i}'] = eval(args, env_cfgs, seed=i) + results[f'seed{i}'] = eval(args, seed=i) compare_temperature(args, results) compare_hp_usage(args, results) diff --git a/scripts/eval_rbc.py b/scripts/eval_rbc.py index a785aab..346415d 100644 --- a/scripts/eval_rbc.py +++ b/scripts/eval_rbc.py @@ -1,17 +1,13 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # LivingLab from livinglab.envs.livinglab_env import LivingLabEnv # Utils +import os import wandb -import json, yaml +import json import argparse import numpy as np from matplotlib import pyplot as plt -from collections import defaultdict -from glob import glob from tabulate import tabulate from agent import HourRBC, ComfortRBC @@ -50,7 +46,9 @@ def parse_arguments(): # LivingLab configs parser.add_argument('--agent', type=str, default='HourRBC', help="RBC agent to evaluete") - parser.add_argument('--env_cfgs', type=str, default='./experiments/PPO_comfort_reward/env_cfgs/test_cfgs.json', help="Path to the configurations of the environment") + parser.add_argument('--env_cfgs', type=str, default='./config/default.json', help="Path to the configurations of the environment") + parser.add_argument('--start', type=int, nargs='?', help="Initial simulation time step") + parser.add_argument('--end', type=int, nargs='?', help="Ending simulation time step") # Wandb logging parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") @@ -115,9 +113,11 @@ def compare_temperature(args, results): bx1.plot(np.zeros_like(cooling_demand), color='black', linestyle='--') bx1.grid('on') - os.makedirs(f'./experiments/{args.agent}/figs', exist_ok=True) - fig.savefig(f'./experiments/{args.agent}/figs/indoor_dry_bulb_temperature.png', format='png') - plt.show() + name = args.agent if args.name is None else args.name + env_name = args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'./experiments/{name}/{env_name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/{env_name}/figs/indoor_dry_bulb_temperature.png', format='png') + def compare_hp_usage(args, results): # Outdoor temperatures @@ -164,9 +164,10 @@ def compare_hp_usage(args, results): bx.legend() bx.grid('on') - os.makedirs(f'./experiments/{args.agent}/figs', exist_ok=True) - fig.savefig(f'./experiments/{args.agent}/figs/mshp_usage.png', format='png') - plt.show() + name = args.agent if args.name is None else args.name + env_name = args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'./experiments/{name}/{env_name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/{env_name}/figs/mshp_usage.png', format='png') def compare_thermal_battery(args, results): @@ -188,9 +189,10 @@ def compare_thermal_battery(args, results): bx2.set_ylim(ymin=-0.05, ymax=1.05) bx2.yaxis.label.set_color('xkcd:orange') - os.makedirs(f'./experiments/{args.agent}/figs', exist_ok=True) - fig.savefig(f'./experiments/{args.agent}/figs/thermal_battery.png', format='png') - plt.show() + name = args.agent if args.name is None else args.name + env_name = args.env_cfgs.split('/')[-1].split('.')[0] + os.makedirs(f'./experiments/{name}/{env_name}/figs', exist_ok=True) + fig.savefig(f'./experiments/{name}/{env_name}/figs/thermal_battery.png', format='png') def log_kpis(args, results): @@ -208,7 +210,7 @@ def log_kpis(args, results): run = wandb.init( entity=args.entity, project='LivingLab_RL_eval_v3' if args.project is None else args.project, - name=args.agent + name=args.agent if args.name is None else args.name ) # Log results @@ -221,7 +223,13 @@ def log_kpis(args, results): def eval(args, env_cfgs): # Load the LivingLabEnv given the configurations - env = LivingLabEnv(**env_cfgs) + env = LivingLabEnv.from_json(config=env_cfgs) + + # Modify start and end time step if provided + if args.start is not None: + env.start_time_step = args.start + if args.end is not None: + env.end_time_step = args.end # Load the agent if args.agent == 'HourRBC': @@ -291,5 +299,5 @@ def eval(args, env_cfgs): compare_temperature(args, results) compare_hp_usage(args, results) - # if args.wandb: - # log_kpis(args, results) \ No newline at end of file + if args.wandb: + log_kpis(args, results) \ No newline at end of file diff --git a/scripts/train.py b/scripts/train.py index eb5f88b..9d8121d 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -1,6 +1,3 @@ -import sys, os; sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import warnings; warnings.filterwarnings("ignore", category=UserWarning) - # Omnisafe from ext import omnisafe @@ -8,6 +5,7 @@ from livinglab.envs.livinglab_env import LivingLabEnv # Utils +import os import torch import argparse import yaml, json @@ -39,6 +37,7 @@ def parse_arguments(): parser.add_argument('--wandb', action='store_true', help="Wandb logging flag") parser.add_argument('--project', type=str, nargs='?', help="Wandb project") parser.add_argument('--entity', type=str, nargs='?', help="Wandb entity") + parser.add_argument('--exp_name', type=str, nargs='?', help="Experiment name") parser.add_argument('--tag', type=str, nargs='*', help="Wandb tag") return parser.parse_args() @@ -88,7 +87,7 @@ def train(args, env_cfgs): 'wandb_project': args.project if args.project is not None else 'None', 'entity': args.entity if args.entity is not None else 'None', 'mode': 'online', - 'tag': list(args.tag) if args.tag is not None else [] + 'tag': [args.exp_name] }, # --- LIVINGLAB KEY ARGUMENTS --- @@ -123,8 +122,8 @@ def train(args, env_cfgs): args = parse_arguments() # Experiment logging - exp_name = datetime.now().strftime('%d-%m-%y_%H:%M') - exp_dir = f'./experiments/{args.algo}_{exp_name}' + exp_name = f"{args.algo}_{datetime.now().strftime('%d-%m-%y_%H:%M')}" if args.exp_name is None else args.exp_name + exp_dir = f'./experiments/{exp_name}' seed_dir = f'{exp_dir}/seed{args.seed}' os.makedirs(seed_dir, exist_ok=True)