From c5de546c2185e3f7584da8f511955765c6ac59bf Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Mon, 14 Apr 2025 15:15:55 +0100 Subject: [PATCH 01/35] prep for new developments --- CHANGELOG.md | 2 ++ brightwind/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e51904e..3b6bafec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Additional labels for pre-release and build metadata are available as extensions --- +## [2.4.0-dev] +1. ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 85e5745d..20152832 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -12,4 +12,4 @@ __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.3.0' +__version__ = '2.4.0-dev' From f596c79540f0550992515d7c2d8f2c3f35971ca8 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Wed, 11 Jun 2025 15:12:34 +0200 Subject: [PATCH 02/35] iss #458 numpy pandas update --- brightwind/analyse/analyse.py | 2 +- brightwind/analyse/plot.py | 8 ++--- brightwind/analyse/shear.py | 2 +- brightwind/load/station.py | 4 +-- brightwind/transform/transform.py | 16 ++++----- requirements.txt | 4 +-- tests/test_correlation.py | 60 +++++++++++++++---------------- tests/test_plot.py | 4 +-- tests/test_transform.py | 40 ++++++++++----------- 9 files changed, 70 insertions(+), 70 deletions(-) diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 1ad79b1e..936efc4e 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -740,7 +740,7 @@ def custom_agg(x): graph = plt.figure(figsize=(15, 8)) ax = graph.add_axes([0.1, 0.1, 0.8, 0.8]) - bw_plt._bar_subplot(distributions.replace([np.inf, -np.inf], np.NAN), x_label=x_label, y_label=aggregation_method, + bw_plt._bar_subplot(distributions.replace([np.inf, -np.inf], np.nan), x_label=x_label, y_label=aggregation_method, max_bar_axis_limit=max_y_value, bin_tick_labels=bin_labels, bar_tick_label_format=bar_tick_label_format, legend=legend, total_width=0.8, ax=ax) plt.close() diff --git a/brightwind/analyse/plot.py b/brightwind/analyse/plot.py index 3f18d642..95303b59 100644 --- a/brightwind/analyse/plot.py +++ b/brightwind/analyse/plot.py @@ -1455,7 +1455,7 @@ def _bar_subplot(data, x_label=None, y_label=None, min_bar_axis_limit=None, max_ var_to_bin_against=data['Spd80mN'].to_frame(), aggregation_method = '%frequency') fig = plt.figure(figsize=(15, 8)) - bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), y_label='%frequency') + bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.nan).dropna(), y_label='%frequency') """ @@ -1638,7 +1638,7 @@ def plot_freq_distribution(data, max_y_value=None, x_tick_labels=None, x_label=N distribution = bw.analyse.analyse._derive_distribution(data['Spd40mN'], var_to_bin_against=data['Spd40mN'], bins=None, aggregation_method = '%frequency').rename('Spd40mN') - bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), + bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.nan).dropna(), max_y_value=None,x_tick_labels=[], x_label=None, y_label='%frequency') @@ -1651,7 +1651,7 @@ def plot_freq_distribution(data, max_y_value=None, x_tick_labels=None, x_label=N aggregation_method='count').rename('Spd80mN') bw.analyse.plot.plot_freq_distribution(pd.concat([distribution1, distribution2], axis=1 - ).replace([np.inf, -np.inf], np.NAN).dropna(), + ).replace([np.inf, -np.inf], np.nan).dropna(), max_y_value=None, x_tick_labels=None, x_label=None, y_label='count', total_width=1, legend=True) @@ -1667,7 +1667,7 @@ def plot_freq_distribution(data, max_y_value=None, x_tick_labels=None, x_label=N fig = plt.figure(figsize=(15, 8)) ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) - _bar_subplot(data.replace([np.inf, -np.inf], np.NAN), x_label=x_label, + _bar_subplot(data.replace([np.inf, -np.inf], np.nan), x_label=x_label, y_label=y_label, max_bar_axis_limit=max_y_value, bin_tick_labels=x_tick_labels, bar_tick_label_format=bar_tick_label_format, legend=legend, total_width=total_width, ax=ax) diff --git a/brightwind/analyse/shear.py b/brightwind/analyse/shear.py index 6cefd1d2..592147f9 100644 --- a/brightwind/analyse/shear.py +++ b/brightwind/analyse/shear.py @@ -1096,7 +1096,7 @@ def _fill_df_12x24(data): idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1H') # create new dataframe with 24 rows only interval number of unique values - df = pd.DataFrame({cols: [np.NaN] for cols in df_copy.columns}, index=pd.DatetimeIndex(idx).time) + df = pd.DataFrame({cols: [np.nan] for cols in df_copy.columns}, index=pd.DatetimeIndex(idx).time) df = pd.concat( [(df[df_copy.index[0].hour:]), (df[:df_copy.index[0].hour])], axis=0) diff --git a/brightwind/load/station.py b/brightwind/load/station.py index 3ed0d25f..3a9bd29c 100644 --- a/brightwind/load/station.py +++ b/brightwind/load/station.py @@ -1165,11 +1165,11 @@ def get_heights(self, names=None, measurement_type_id=None): name_found = True break elif meas_point['name'] == name and meas_point.get('height_m') is None: - heights.append(np.NaN) + heights.append(np.nan) name_found = True break if name_found is False: - heights.append(np.NaN) + heights.append(np.nan) return heights diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 1c17e7f3..1d301593 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -240,18 +240,18 @@ def _get_overlapping_data(df1, df2, averaging_prd=None): # averaging will start from this timestamp. if not (df2.index == start).any(): if type(df2) == pd.DataFrame: - df2 = pd.concat([df2, pd.DataFrame({cols: [np.NaN] for cols in df2.columns}, + df2 = pd.concat([df2, pd.DataFrame({cols: [np.nan] for cols in df2.columns}, index=[pd.to_datetime(start)])]) else: - df2[pd.to_datetime(start)] = np.NaN + df2[pd.to_datetime(start)] = np.nan df2.sort_index(inplace=True) if not (df1.index == start).any(): - # df1.loc[pd.to_datetime(start)] = np.NaN + # df1.loc[pd.to_datetime(start)] = np.nan if type(df1) == pd.DataFrame: - df1 = pd.concat([df1, pd.DataFrame({cols: [np.NaN] for cols in df1.columns}, + df1 = pd.concat([df1, pd.DataFrame({cols: [np.nan] for cols in df1.columns}, index=[pd.to_datetime(start)])]) else: - df1[pd.to_datetime(start)] = np.NaN + df1[pd.to_datetime(start)] = np.nan df1.sort_index(inplace=True) return df1[start:], df2[start:] @@ -538,7 +538,7 @@ def _vector_avg_of_wdirs_dataframe(wdirs, wspds=None): # means there is no wind direction => return NaN nan_mask = (avg_dir_df['sine'] == 0) & (avg_dir_df['cosine'] == 0) avg_dir_df['avg_dir'] = np.rad2deg(np.arctan2(sine, cosine)) % 360 - avg_dir_df['avg_dir'][nan_mask] = np.NaN + avg_dir_df['avg_dir'][nan_mask] = np.nan return avg_dir_df['avg_dir'] @@ -582,7 +582,7 @@ def _vector_avg_of_wdirs_list(wdirs, wspds=None): wspds = a[1] # if the resulting wdir array is empty, return NAN if wdirs.size == 0: - return np.NaN + return np.nan if wspds is None: sine = np.mean(np.round(np.sin(np.deg2rad(wdirs)), 5)) # sin of each angle, East component @@ -594,7 +594,7 @@ def _vector_avg_of_wdirs_list(wdirs, wspds=None): # If both sine and cosine result in zero then all the directions cancel and you end up where you started which # means there is no wind direction => return NaN if sine == 0 and cosine == 0: - avg_dir = np.NaN + avg_dir = np.nan else: avg_dir = np.rad2deg(np.arctan2(sine, cosine)) % 360 if avg_dir == 360.0: # preference to have 0 returned instead of 360 diff --git a/requirements.txt b/requirements.txt index 337e20f7..8be85f60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pandas >= 0.24.0, <=2.0.1 -numpy >= 1.16.4, <2.0.0 +pandas >= 0.24.0 +numpy >= 1.16.4 scikit-learn >= 0.19.1 matplotlib >= 3.0.3 requests >= 2.20.0 diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 9e247592..62964252 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -5,15 +5,15 @@ import warnings wndspd = 8 -wndspd_df = pd.DataFrame([2, 13, np.NaN, 5, 8]) -wndspd_series = pd.Series([2, 13, np.NaN, 5, 8]) +wndspd_df = pd.DataFrame([2, 13, np.nan, 5, 8]) +wndspd_series = pd.Series([2, 13, np.nan, 5, 8]) current_slope = 0.045 current_offset = 0.235 new_slope = 0.046 new_offset = 0.236 wndspd_adj = 8.173555555555556 -wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) -wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) +wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) +wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) DATA = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) DATA_CLND = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) @@ -209,62 +209,62 @@ def test_synthesize(): '2017-06-05 00:00:00': 9.88184374999999, '2017-06-15 00:00:00': 7.639711111111101, '2017-06-25 00:00:00': 9.144633101851865, - '2017-07-05 00:00:00': np.NaN, - '2017-07-15 00:00:00': np.NaN, + '2017-07-05 00:00:00': np.nan, + '2017-07-15 00:00:00': np.nan, '2017-07-25 00:00:00': 7.127115740740741, '2017-08-04 00:00:00': 6.4158555555555505, '2017-08-14 00:00:00': 7.510521527777763, '2017-08-24 00:00:00': 5.943415798611104, - '2017-09-03 00:00:00': np.NaN, - '2017-09-13 00:00:00': np.NaN, + '2017-09-03 00:00:00': np.nan, + '2017-09-13 00:00:00': np.nan, '2017-09-23 00:00:00': 12.601222222222226, '2017-10-03 00:00:00': 9.425352777777784, '2017-10-13 00:00:00': 9.466307638888878, '2017-10-23 00:00:00': 8.84323971518988}} - result_ord_lst_sq_dir = {'Spd80mN_Synthesized': {'2016-03-01': np.NaN, + result_ord_lst_sq_dir = {'Spd80mN_Synthesized': {'2016-03-01': np.nan, '2016-04-01': 6.598875, - '2016-05-01': np.NaN, + '2016-05-01': np.nan, '2016-06-01': 5.108156, '2016-07-01': 6.319782, '2016-08-01': 7.093956, - '2016-09-01': np.NaN, + '2016-09-01': np.nan, '2016-10-01': 6.669446, - '2016-11-01': np.NaN, + '2016-11-01': np.nan, '2016-12-01': 8.900778, '2017-01-01': 9.501281, '2017-02-01': 9.134509, - '2017-03-01': np.NaN, + '2017-03-01': np.nan, '2017-04-01': 7.783390, - '2017-05-01': np.NaN, + '2017-05-01': np.nan, '2017-06-01': 8.525249, '2017-08-01': 6.715885, '2017-10-01': 9.479016}} - result_speed_sort = {'Spd80mN_Synthesized': {'2016-03-01': np.NaN, + result_speed_sort = {'Spd80mN_Synthesized': {'2016-03-01': np.nan, '2016-04-01': 6.598875, - '2016-05-01': np.NaN, + '2016-05-01': np.nan, '2016-06-01': 5.108156, - '2016-07-01': np.NaN, + '2016-07-01': np.nan, '2016-08-01': 7.093956, - '2016-09-01': np.NaN, + '2016-09-01': np.nan, '2016-10-01': 6.669446, - '2016-11-01': np.NaN, + '2016-11-01': np.nan, '2016-12-01': 8.900778, - '2017-01-01': np.NaN, + '2017-01-01': np.nan, '2017-02-01': 9.134509, - '2017-03-01': np.NaN}, - 'Dir78mS_Synthesized': {'2016-03-01': np.NaN, + '2017-03-01': np.nan}, + 'Dir78mS_Synthesized': {'2016-03-01': np.nan, '2016-04-01': 318.639591, - '2016-05-01': np.NaN, + '2016-05-01': np.nan, '2016-06-01': 129.720897, - '2016-07-01': np.NaN, + '2016-07-01': np.nan, '2016-08-01': 235.377543, '2016-09-01': 223.965222, '2016-10-01': 114.451746, - '2016-11-01': np.NaN, + '2016-11-01': np.nan, '2016-12-01': 219.815444, '2017-01-01': 237.667123, '2017-02-01': 197.538174, - '2017-03-01': np.NaN}} + '2017-03-01': np.nan}} data_spd80mn_even_months = DATA_CLND['Spd80mN'][DATA_CLND.index.month.isin([2, 4, 6, 8, 10, 12])] # Test the synthesise for when the target data starts before the reference data. @@ -305,10 +305,10 @@ def test_synthesize(): # Test the synthesise when SpeedSort correlation is used using 10 min averaging period. data_test = DATA_CLND[['Spd80mN', 'Spd60mN', 'Dir78mS', 'Dir58mS']].copy() - data_test['Dir78mS']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.NaN - data_test['Spd80mN']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.NaN - data_test['Dir58mS']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.NaN - data_test['Spd60mN']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.NaN + data_test['Dir78mS']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan + data_test['Spd80mN']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan + data_test['Dir58mS']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan + data_test['Spd60mN']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan ss_cor = bw.Correl.SpeedSort(data_test['Spd80mN'], data_test['Dir78mS'], data_test['Spd60mN'], data_test['Dir58mS'], averaging_prd='10min') ss_cor.run() diff --git a/tests/test_plot.py b/tests/test_plot.py index 68bf839b..235f935c 100644 --- a/tests/test_plot.py +++ b/tests/test_plot.py @@ -184,7 +184,7 @@ def test_bar_subplot(): var_to_bin_against=DATA['Spd80mN'].to_frame(), aggregation_method='%frequency') fig = plt.figure(figsize=(15, 8)) - bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), y_label='%frequency') + bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.nan).dropna(), y_label='%frequency') assert True @@ -194,7 +194,7 @@ def test_plot_freq_distribution(): distribution = bw.analyse.analyse._derive_distribution(DATA['Spd40mN'], var_to_bin_against=DATA['Spd40mN'], bins=None, aggregation_method='%frequency').rename('Spd40mN') - bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), + bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.nan).dropna(), max_y_value=None, x_tick_labels=[], x_label=None, y_label='%frequency') diff --git a/tests/test_transform.py b/tests/test_transform.py index d26d916d..7e3f176d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -14,15 +14,15 @@ wndspd = 8 -wndspd_df = pd.DataFrame([2, 13, np.NaN, 5, 8]) -wndspd_series = pd.Series([2, 13, np.NaN, 5, 8]) +wndspd_df = pd.DataFrame([2, 13, np.nan, 5, 8]) +wndspd_series = pd.Series([2, 13, np.nan, 5, 8]) current_slope = 0.045 current_offset = 0.235 new_slope = 0.046 new_offset = 0.236 wndspd_adj = 8.173555555555556 -wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) -wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) +wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) +wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) ref_date = pd.to_datetime('2000-01-01') DATA = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) @@ -50,13 +50,13 @@ def test_selective_avg(): days = pd.date_range(date_today, date_today + datetime.timedelta(24), freq='D') data = pd.DataFrame({'DTM': days}) data = data.set_index('DTM') - data['Spd1'] = [1, np.NaN, 1, 1, 1, 1, 1, 1, 1, np.NaN, 1, 1, 1, 1, np.NaN, 1, 1, np.NaN, 1, 1, 1, 1, np.NaN, 1, 1] - data['Spd2'] = [2, 2, np.NaN, 2, 2, 2, 2, 2, np.NaN, 2, 2, 2, 2, np.NaN, 2, 2, 2, np.NaN, 2, 2, 2, 2, 2, np.NaN, 2] - data['Dir'] = [0, 15, 30, 45, np.NaN, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, - 315, np.NaN, 345, 360] + data['Spd1'] = [1, np.nan, 1, 1, 1, 1, 1, 1, 1, np.nan, 1, 1, 1, 1, np.nan, 1, 1, np.nan, 1, 1, 1, 1, np.nan, 1, 1] + data['Spd2'] = [2, 2, np.nan, 2, 2, 2, 2, 2, np.nan, 2, 2, 2, 2, np.nan, 2, 2, 2, np.nan, 2, 2, 2, 2, 2, np.nan, 2] + data['Dir'] = [0, 15, 30, 45, np.nan, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, + 315, np.nan, 345, 360] # Test Case 1: Neither boom is near 0-360 crossover - result = np.array([1.5, 2, 1, 1.5, 1.5, 1.5, 1.5, 2, 1, 2, 2, 2, 1.5, 1, 2, 1.5, 1.5, np.NaN, + result = np.array([1.5, 2, 1, 1.5, 1.5, 1.5, 1.5, 2, 1, 2, 2, 2, 1.5, 1, 2, 1.5, 1.5, np.nan, 1.5, 1, 1, 1, 2, 1, 1.5]) bw.selective_avg(data[['Spd1']], data[['Spd2']], data[['Dir']], boom_dir_1=315, boom_dir_2=135, sector_width=60) @@ -65,21 +65,21 @@ def test_selective_avg(): assert np_array_equal(sel_avg, result) # Test Case 2: Boom 1 is near 0-360 crossover - result = np.array([1.0, 2.0, 1.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, 2.0, 1.0, 2.0, 2.0, 1.5, np.NaN, + result = np.array([1.0, 2.0, 1.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, 2.0, 1.0, 2.0, 2.0, 1.5, np.nan, 1.5, 1.5, 1.5, 1.5, 2.0, 1.0, 1.0]) sel_avg = np.array(bw.selective_avg(data.Spd1, data.Spd2, data.Dir, boom_dir_1=20, boom_dir_2=200, sector_width=60)) assert np_array_equal(sel_avg, result) # Test Case 3: Boom 2 is near 0-360 crossover - result = np.array([2.0, 2.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.5, 1.5, np.NaN, + result = np.array([2.0, 2.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.5, 1.5, np.nan, 1.5, 1.5, 1.5, 1.5, 2.0, 1.0, 2.0]) sel_avg = np.array(bw.selective_avg(data.Spd1, data.Spd2, data.Dir, boom_dir_1=175, boom_dir_2=355, sector_width=60)) assert np_array_equal(sel_avg, result) # Test Case 4: Booms at 90 deg to each other - result = np.array([1.0, 2.0, 1.0, 1.5, 1.5, 2.0, 2.0, 2.0, 1.0, 2.0, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, np.NaN, + result = np.array([1.0, 2.0, 1.0, 1.5, 1.5, 2.0, 2.0, 2.0, 1.0, 2.0, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, np.nan, 1.5, 1.5, 1.5, 1.5, 2.0, 1.0, 1.0]) sel_avg = np.array(bw.selective_avg(data.Spd1, data.Spd2, data.Dir, boom_dir_1=270, boom_dir_2=180, sector_width=60)) @@ -161,13 +161,13 @@ def test_offset_wind_direction_float(): def test_offset_wind_direction_df(): - wdir_df_offset = pd.DataFrame([355, 15, np.NaN, 25, 335]) - assert wdir_df_offset.equals(bw.offset_wind_direction(pd.DataFrame([10, 30, np.NaN, 40, 350]), 345)) + wdir_df_offset = pd.DataFrame([355, 15, np.nan, 25, 335]) + assert wdir_df_offset.equals(bw.offset_wind_direction(pd.DataFrame([10, 30, np.nan, 40, 350]), 345)) def test_offset_wind_direction_series(): - wdir_series_offset = pd.Series([355, 15, np.NaN, 25, 335]) - assert wdir_series_offset.equals(bw.offset_wind_direction(pd.Series([10, 30, np.NaN, 40, 350]), 345)) + wdir_series_offset = pd.Series([355, 15, np.nan, 25, 335]) + assert wdir_series_offset.equals(bw.offset_wind_direction(pd.Series([10, 30, np.nan, 40, 350]), 345)) def test_apply_wind_vane_dead_band_offset(): @@ -443,10 +443,10 @@ def test_average_wdirs(): assert bw.average_wdirs(wdirs) == 0.0 wdirs = np.array([0, 180]) - assert bw.average_wdirs(wdirs) is np.NaN + assert bw.average_wdirs(wdirs) is np.nan wdirs = np.array([90, 270]) - assert bw.average_wdirs(wdirs) is np.NaN + assert bw.average_wdirs(wdirs) is np.nan wdirs = np.array([45, 135]) assert bw.average_wdirs(wdirs) == 90 @@ -455,7 +455,7 @@ def test_average_wdirs(): assert bw.average_wdirs(wdirs) == 180 wdirs = np.array([45, 315, 225, 135]) - assert bw.average_wdirs(wdirs) is np.NaN + assert bw.average_wdirs(wdirs) is np.nan wdirs = np.array([225, 315]) assert bw.average_wdirs(wdirs) == 270 @@ -473,7 +473,7 @@ def test_average_wdirs(): assert round(bw.average_wdirs(wdirs_with_nan, wspds_with_nan), 3) == 15.0 wspds_with_nan = [np.nan, np.nan, np.nan] - assert bw.average_wdirs(wdirs_with_nan, wspds_with_nan) is np.NaN + assert bw.average_wdirs(wdirs_with_nan, wspds_with_nan) is np.nan wspds_with_nan = [3, 4, np.nan] assert round(bw.average_wdirs(pd.Series(wdirs_with_nan), pd.Series(wspds_with_nan)), 3) == 15.0 From 453b62326ad643645667dc9283f2d353384a3c58 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:12:37 +0200 Subject: [PATCH 03/35] loosen pandas numpy requirements in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bad46d06..603b3e52 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ def get_version(rel_path): keywords=['BRIGHT', 'WIND', 'RESOURCE', 'DATA', 'ANALYSTS', 'PROCESSING', 'WASP', 'ROSE', 'WINDFARMER', 'OPENWIND', 'WIND PRO', 'WINDOGRAPHER'], install_requires=[ - 'pandas>=0.24.0, <=2.0.1', - 'numpy>=1.16.4, <2.0.0', + 'pandas>=0.24.0', + 'numpy>=1.16.4', 'scikit-learn>=0.19.1', 'matplotlib>=3.0.3', 'requests>=2.20.0', From b7c793a4f2bc083a40550d496816178f6b90bce5 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:53:58 +0200 Subject: [PATCH 04/35] add jinja2 to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8be85f60..e6c8147e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ ipython >= 7.4.0 gmaps >= 0.9.0 colormap >= 1.0.1 easydev >= 0.10.0 -jsonschema >= 4.17.3 \ No newline at end of file +jsonschema >= 4.17.3 +jinja2 >= 3.0.0 \ No newline at end of file From d314ed8947c88644305b9f4de6940bcbee083413 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Fri, 13 Jun 2025 13:01:04 +0200 Subject: [PATCH 05/35] iss #437 add env to test.yml --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1c512661..2fda0033 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,10 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +env: + BRIGHTHUB_EMAIL: email + BRIGHTHUB_PASSWORD: password + jobs: build: From 3b871ef982db2f01adfa53fd62f4548de4659a1c Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:07:31 +0200 Subject: [PATCH 06/35] add github secrets --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2fda0033..c6511c45 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,8 +10,8 @@ on: workflow_dispatch: env: - BRIGHTHUB_EMAIL: email - BRIGHTHUB_PASSWORD: password + BRIGHTHUB_EMAIL: ${{ secrets.BRIGHTHUB_EMAIL }} + BRIGHTHUB_PASSWORD: ${{ secrets.BRIGHTHUB_PASSWORD }} jobs: build: From b4c62386e73cf54123b6d25eea56ab59babb0f9e Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 11:51:23 +0100 Subject: [PATCH 07/35] iss #458 and #407 fixed bug with pandas v2.0 --- brightwind/load/load.py | 2 +- tests/test_load.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/brightwind/load/load.py b/brightwind/load/load.py index d36f4206..32e344fd 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -88,7 +88,7 @@ def _assemble_df_from_folder(source_folder, file_type, function_to_get_df, print assembled_df = pd.DataFrame() for file_name in files_list: df = function_to_get_df(file_name, **kwargs) - assembled_df = assembled_df.append(df, verify_integrity=True) + assembled_df = pd.concat([assembled_df, df], verify_integrity=True) if print_progress: print("{0} file read and appended".format(file_name)) ctr = ctr + 1 diff --git a/tests/test_load.py b/tests/test_load.py index 0cf480fd..7332d240 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -4,6 +4,7 @@ import pandas as pd import numpy as np import json +from unittest.mock import patch DEMO_DATA_FOLDER = os.path.join(os.path.dirname(__file__), '../brightwind/demo_datasets') @@ -123,6 +124,19 @@ def test_load_csv(): data3['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999)).all().all() assert (data['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999) == data4['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999)).all().all() + + # test loading files from folder + bw.export_csv(data[:'2016-01-09 17:00'], os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv')) + bw.export_csv(data['2016-01-09 17:10':'2016-01-09 18:00'], os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')) + with patch('brightwind.load.load._list_files', return_value=[ + os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv'), + os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')]): + assert isinstance(bw.load_csv(DEMO_DATA_FOLDER, '.csv'), pd.DataFrame) + # Remove the temp files + if os.path.exists(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv')): + os.remove(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv')) + if os.path.exists(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')): + os.remove(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')) def test_load_windographer_txt(): From 128383c05528de84ce990157936bfaa569f35d3e Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 12:12:37 +0100 Subject: [PATCH 08/35] iss #458 minor formatting change --- brightwind/load/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 32e344fd..01f85d76 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -2442,7 +2442,7 @@ def apply_cleaning_rules(data, cleaning_rules_file_or_list, inplace=False, repla if utils.is_file_extension(cleaning_rules_file_or_list, ".json"): with open(cleaning_rules_file_or_list) as file: cleaning_json = json.load(file) - elif isinstance(cleaning_rules_file_or_list, List): + elif isinstance(cleaning_rules_file_or_list, list): if not all(isinstance(item, dict) for item in cleaning_rules_file_or_list): raise TypeError("All elements in the `cleaning_rules_file_or_list` must be dictionaries.") cleaning_json = cleaning_rules_file_or_list From 0e78ef5c69a47cea52cd0cfef496e2117552c4ed Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 12:55:46 +0100 Subject: [PATCH 09/35] iss #458 updated max requirements for pandas and python --- requirements.txt | 2 +- setup.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e6c8147e..2d0f8a09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pandas >= 0.24.0 +pandas >=0.24.0, <3.0.0, numpy >= 1.16.4 scikit-learn >= 0.19.1 matplotlib >= 3.0.3 diff --git a/setup.py b/setup.py index 603b3e52..595c573c 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ def get_version(rel_path): keywords=['BRIGHT', 'WIND', 'RESOURCE', 'DATA', 'ANALYSTS', 'PROCESSING', 'WASP', 'ROSE', 'WINDFARMER', 'OPENWIND', 'WIND PRO', 'WINDOGRAPHER'], install_requires=[ - 'pandas>=0.24.0', + 'pandas>=0.24.0, <3.0.0', 'numpy>=1.16.4', 'scikit-learn>=0.19.1', 'matplotlib>=3.0.3', @@ -67,6 +67,8 @@ def get_version(rel_path): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: MIT License", ], ) From b90f6f15479d9c3e529fe2b3bb0d137fd202ab29 Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 13:11:05 +0100 Subject: [PATCH 10/35] iss #458 updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b6bafec..efbc6df5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0-dev] -1. +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. From 69610ad80020ec56fa975c786e2a522974c67cb8 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Thu, 17 Jul 2025 15:22:58 +0100 Subject: [PATCH 11/35] remove python 3.7 as no longer supported by ubuntu 24.04 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c6511c45..165ac99d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From dac6713e6e0d7fb5ad97ee45ffe3138ccd216716 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:29:01 +0100 Subject: [PATCH 12/35] [Iss531] add function to extrapolate pressure (SR8BM2.5) (#537) * iss #531 added lapse_prs function * iss #531 updated docstring of lapse_prs * iss #531 updated function and argument names and docstring * iss #531 added tests * iss #531 updated changelog * iss #531 updated function name to extrapolate_pressure * iss #531 updated function name to extrapolate_pressure in changelog * iss #531 addressed comments, renamed to scale_pressure_to_height * iss #531 formatting fixes * iss #531 updated names to include air pressure/air temp * iss #531 moved constants to constants.py file * iss #531 updated function to include type declaration * iss #531 removed constants.py and re-added capitalised constants to analyse.py * iss #531 updated formatting and few rewording * iss #531 add error for type and input length and test * iss #531 minor format change * iss #531 update doc string * iss #531 format updates to changelog --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 7 +++ brightwind/analyse/analyse.py | 96 ++++++++++++++++++++++++++++++++++- tests/test_analyse.py | 38 ++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efbc6df5..de7677f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.4.0-dev] 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +### Bug Fixes +1. + +### New Features and Enhancements +1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) + + ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. Key improvements include more reliable wind and solar data handling, expanded plotting capabilities (including diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 936efc4e..31be51cb 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -11,6 +11,7 @@ import warnings import textwrap from matplotlib.ticker import PercentFormatter +from typing import Union __all__ = ['monthly_means', 'momm', @@ -27,7 +28,17 @@ 'basic_stats', 'TI', 'sector_ratio', - 'calc_air_density'] + 'calc_air_density', + 'scale_air_pressure_to_height'] + +# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere +ACCEL_DUE_TO_GRAVITY = 9.80665 + +# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere +TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + +# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere +GAS_CONST_DRY_AIR = 287.05 def dist_matrix(var_series, x_series, y_series, @@ -2065,3 +2076,86 @@ def calc_air_density(temperature, pressure, elevation_ref=None, elevation_site=N raise TypeError('elevation_ref should be a number') else: return ref_air_density + + +def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], + ref_air_temp_degC: Union[float, pd.Series], + ref_height_m: Union[float, int], + target_height_m: Union[float, int]) -> Union[float, int, pd.Series]: + """ + Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) + and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). + + Calculation based on ISO:2533-1975 Standard Atmosphere (https://www.iso.org/obp/ui/#iso:std:iso:2533:en) + as suggested by IEC 61400-12-1: + + scaled_air_pressure = ref_air_pressure_hPa*((1 + (L/ref_air_temp_K)*(target_height_m - ref_height_m))**(-g/(L*R))) + where: + g = 9.80665 is acceleration due to gravity (m/s^2) + L = -0.0065 is the temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = 287.05 is the specific gas constant for dry air (J/K/kg or m2/K/s2) + + :param ref_air_pressure_hPa: Reference air pressure value(s) in hPa (1mbar = 1hPa = 100Pa). + :type ref_air_pressure_hPa: float or int or pandas.Series + :param ref_air_temp_degC: Reference air temperature value(s) in degrees celsius. + :type ref_air_temp_degC: float or int or pandas.Series + :param ref_height_m: Height (in metres) of reference air temperature (ref_air_temp_degC) + and air pressure (ref_air_pressure_hPa). + :type ref_height_m: float or int + :param target_height_m: Height (in metres) which ref_air_pressure_hPa is scaled to. + :type target_height_m float or int + :return: Air pressure at specified height of target_height_m in hPa (1mbar = 1hPa = 100Pa). + Type depends on type(ref_air_pressure_hPa) and type(ref_air_temp_degC) inputs. + :rtype: float or int or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale float value of air pressure + round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, ref_height_m=10, + target_height_m=200), 2) + # 977.45 + + # scale air pressure based on input series of reference air pressure and air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_pressure_to_height(ref_air_pressure_hPa=data['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=data['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10).round(2) + # Timestamp + # 2016-01-09 17:10:00 933.07 + # 2016-01-09 17:20:00 933.07 + # 2016-01-09 17:30:00 933.07 + # 2016-01-09 17:40:00 932.07 + # 2016-01-09 17:50:00 932.07 + # 2016-01-09 18:00:00 932.07 + # dtype: float64 + + """ + + # check input types + for var, var_name in zip([ref_air_pressure_hPa, ref_air_temp_degC], ['ref_air_pressure_hPa', 'ref_air_temp_degC']): + if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series)): + raise TypeError(f"{var_name} must be a float or int or pandas.Series.") + for var, var_name in zip([ref_height_m, target_height_m], ['ref_height_m', 'target_height_m']): + if not (isinstance(var, float) or isinstance(var, int)): + raise TypeError(f"{var_name} must be a float or int.") + + # check dimensions of ref_air_pressure_hPa and ref_air_temp_degC if Series + if isinstance(ref_air_pressure_hPa, pd.Series) and (isinstance(ref_air_temp_degC, pd.Series)): + if len(ref_air_pressure_hPa) != len(ref_air_temp_degC): + raise ValueError("ref_air_pressure_hPa and ref_air_temp_degC must have the same dimensions.") + + # Constants as outlined in ISO:2533 + g = ACCEL_DUE_TO_GRAVITY # Acceleration due to gravity (m/s^2) + L = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE # Temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = GAS_CONST_DRY_AIR # Specific gas constant dry air + + ref_air_temp_K = ref_air_temp_degC + 273.15 # Convert temp units to K + + scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) + ) ** (-g / (L * R))) + + return scaled_air_pressure_hPa diff --git a/tests/test_analyse.py b/tests/test_analyse.py index e6a5bb9d..c1857de5 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -659,3 +659,41 @@ def test_dist_matrix_by_direction_sector(): var_to_bin_by_array=[-8, -5, 5, 10, 15, 20, 26], sectors=8) assert True + +def test_scale_air_pressure_to_height(): + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=200), 2)== 977.45 + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=15), 2) == 999.4 + pd.testing.assert_series_equal(bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=DATA['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=DATA['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data = [933.07, 933.07, 933.07, 932.07, 932.07, 932.07], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + # test error raising for invalid input types + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa='invalid_type', + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m=10) + + # test error raising for invalid slope type + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=1000, + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m='invalid_type') + + # test error raising for mismatched dimensions with Series + with pytest.raises(ValueError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=pd.Series([1, 2, 3]), + ref_air_temp_degC=pd.Series([1, 2]), + ref_height_m=2, + target_height_m=10) From ea976845805b5f6f5b32f9b207eca9fc6e821a0a Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:32:38 +0100 Subject: [PATCH 13/35] [Iss530] Add functions to extrapolate temperature and air density (SR14BM9) (#536) * iss #530 added lapse_temp function * iss #530 updated tests and docstring * iss #530 updated docstring * iss #530 minor docstring update * iss #531 added tests and updated function name to lapse_temperature * Revert "iss #531 added tests and updated function name to lapse_temperature" This reverts commit d41ec28e0b4f7b9e1ecf6f2ff85b8312cbe196e4. * iss #530 updated function name to lapse_temperature * iss #530 added tests * iss #534 added lapse_density function * iss #534 added tests * iss #530 updated changelog * iss #534 updated changelog * iss #530 added private function for lapsing, bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 updated name to extrapolate_temperature and updated to use bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 changed lapse_density to extrapolate_density * iss #530 addressed PR comments, updated function names * iss #530 small formatting fixes * iss #530 updated reference to ref for consistency with other PR * iss #530 formatting and docstring and naming fixes * iss #530 updated functions to include type for each argument * iss 530 changed linear funct to public and fixed several typos and example issues * iss 530 fixed tests for utils and analyse * iss 530 fixed typo changelog * iss 530 updated linear_transform to raise errors and added tests * iss 530 added more tests * iss #530 added source of air density lapse rate to docstring * iss #530 include reference for air density lapse rate in docstring * iss #530 updated input to call global variable * iss #530 update docstring of scale_air_density * iss #530 fix PEP 8 warnings * iss #530 update scale_air_temp docstring * iss #530 updates to linear_transform * iss #530 fix typo, update eg and update docstring * Update CHANGELOG.md --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 4 +- brightwind/analyse/analyse.py | 131 +++++++++++++++++++++++++++++++++- brightwind/utils/utils.py | 89 ++++++++++++++++++++++- tests/test_analyse.py | 45 ++++++++++++ tests/test_utils.py | 56 +++++++++++++++ 5 files changed, 322 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de7677f1..54ec8e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ Additional labels for pre-release and build metadata are available as extensions 1. ### New Features and Enhancements -1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) +1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) +1. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([[#534](https://github.com/brightwind-dev/brightwind/issues/534)]) +1. Added `scale_air_temperature_to_height` to output a air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([[#530](https://github.com/brightwind-dev/brightwind/issues/530)]) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 31be51cb..3dbd5fb0 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -29,7 +29,9 @@ 'TI', 'sector_ratio', 'calc_air_density', - 'scale_air_pressure_to_height'] + 'scale_air_pressure_to_height', + 'scale_air_density_to_height', + 'scale_air_temperature_to_height'] # Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere ACCEL_DUE_TO_GRAVITY = 9.80665 @@ -2159,3 +2161,130 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], ) ** (-g / (L * R))) return scaled_air_pressure_hPa + + +def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_kg_m3_m: float = -0.000113) -> Union[float, pd.Series]: + """ + Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height + (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. + + :param ref_air_density_kg_m3: Reference air density value(s) in kg/m3. + :type ref_air_density_kg_m3: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_density_kg_m3. + :type ref_height_m float + :param target_height_m: Height (in metres) that ref_air_density_kg_m3 is scaled to. + :type target_height_m: float + :param lapse_rate_kg_m3_m: Lapse rate describes how air density changes with increasing height above the + earth's surface in kg/m3/m. + Default value of -0.113 kg/m3 per km above earth's surface (-0.000113 kg/m3/m) + taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014). + :type lapse_rate_kg_m3_m: float + :return: Air density at specified height of target_height_m in kg/m3. Type depends on + type(ref_air_density_kg_m3) input. + :rtype: float or pandas.Series + + **Example usage** + :: + import brightwind as bw + + # scale float value of air density using default lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) + # 1.22174 + + # scale float value of air density using non-default value for lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) + # 1.22 + + # derive air density and scale based on series input values for reference air density + data = bw.load_csv(bw.demo_datasets.demo_data) + test_density = bw.calc_air_density(data.T2m, data.P2m) + bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10) + + Timestamp + 2016-01-09 17:10:00 1.186780 + 2016-01-09 17:20:00 1.187175 + 2016-01-09 17:30:00 1.187747 + 2016-01-09 17:40:00 1.185950 + 2016-01-09 17:50:00 1.186301 + 2016-01-09 18:00:00 1.185686 + dtype: float64 + """ + + scaled_air_density = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) + + return scaled_air_density + + +def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_deg_m: float = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE + ) -> Union[float, pd.Series]: + """ + Linearly scales reference air temperature measurement (ref_air_temperature) from its measurement height + (ref_height_m) to the height specified as target_height_m, by applying the constant lapse_rate_deg_m. + + :param ref_air_temperature: Air temperature value(s) in degrees [for example in Celsius or Kelvin]. + :type ref_air_temperature: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_temperature. + :type ref_height_m: float + :param target_height_m: Height (in metres) that ref_air_temperature is scaled to. + :type target_height_m: float + :param lapse_rate_deg_m: Lapse rate describes how air temperature changes with increasing height + above the earth's surface. + Units should be degrees of temperature per unit of height, e.g. °C/m or K/m. + Default value of -6.5 degrees Celsius per km above the earth's surface + (or -0.0065 °C/m) is commonly used as an approximation of the + atmospheric lapse rate. + In particular, the IEC standards rely on the ISO2533:1975 Standard Atmosphere + which states that a lapse rate of 6.5 K/km is valid for geopotential altitudes + of up to 11 km above earth's surface. + This value was also adopted in WASP 11: + Mortensen, N. G., Heathfield, D. N., Rathmann, O., & Nielsen, M. (2014). + Wind Atlas Analysis and Application Program: WAsP 11 Help Facility. Computer + programme, Department of Wind Energy, Technical University of Denmark + https://orbit.dtu.dk/en/publications/wind-atlas-analysis-and-application-program-wasp-11-help-facility + :type lapse_rate_deg_m: float + :return: Air temperature at specified height of target_height_m in same unit as input + ref_air_temperature [for example in Celsius or Kelvin]. Output type depends on + type(ref_air_temperature) input. + :rtype: float or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale air temperature based on float input value for reference air temperature + bw.scale_air_temperature_to_height(ref_air_temperature=10.0065, ref_height_m=10, target_height_m=11) + # 10.0 + + # scale air temperature based on float input value for reference air temperature with non-default lapse_rate_deg_m + bw.scale_air_temperature_to_height(ref_air_temperature=10, ref_height_m=12, target_height_m=10, + lapse_rate_deg_m=-0.001) + # 10.002 + + # scale air temperature based on series input values for reference air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_temperature_to_height(ref_air_temperature=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=20) + # Timestamp + # 2016-01-09 17:10:00 0.837 + # 2016-01-09 17:20:00 0.746 + # 2016-01-09 17:30:00 0.614 + # 2016-01-09 17:40:00 0.735 + # 2016-01-09 17:50:00 0.654 + # 2016-01-09 18:00:00 0.796 + # Name: T2m, dtype: float64 + """ + + scaled_air_temp = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_temperature, slope=lapse_rate_deg_m) + return scaled_air_temp diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index 410944e4..bd6d6361 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -3,12 +3,14 @@ import os import json from jsonschema import Draft7Validator +from typing import Union __all__ = ['slice_data', 'validate_coverage_threshold', 'is_file', 'is_file_extension', - 'validate_json'] + 'validate_json', + 'linear_transform'] def _range_0_to_360(direction): @@ -252,3 +254,88 @@ def validate_json(json_to_check, schema): print(f"Failed schema part: {error.get('schema_path')}\n") return data_is_valid + + +def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], + x_ref: Union[float, int, np.ndarray, pd.Series], + y_ref: Union[float, int, np.ndarray, pd.Series], + slope: Union[float, int]) -> Union[float, int, np.ndarray, pd.Series]: + """ + Perform a linear transform of known (x_ref, y_ref) to calculate y_target, + using a constant slope and known value(s) of x_target. + + Function applies a linear transformation based on the equation of a straight line with constant slope: + y_target = slope * (x_target - x_ref) + y_ref, + where (x_ref, y_ref) is effectively considered a known point on a line with the inputted slope. + + Note that if pd.Series or np.ndarray inputs are provided for x_target, x_ref and y_ref, + then the transform is performed on a point by point basis, as in, the n^{th} value of + the resulting y_target would be: + y_target[n] = slope * (x_target[n] - x_ref[n]) + y_ref[n], + where (x_ref[n], y_ref[n]) is effectively considered a known point on a line with the inputted slope. + Therefore all three of x_target, x_ref and y_ref must have the same dimensions. + The inputted slope is used even in the case of pd.Series or np.ndarray inputs as the purpose of this function + is to perform a linear transformation using a predefined slope, not to fit a line to the input data. + + :param x_target: Target x value(s) at which to calculate y_target. + :type x_target: float or int or numpy.ndarray or pandas.Series + :param x_ref: Reference x value(s) of known reference point(s) on the line. + :type x_ref: float or int or numpy.ndarray or pandas.Series + :param y_ref: Reference y value(s) of known reference point(s) on the line. + :type y_ref: float or int or numpy.ndarray or pandas.Series + :param slope: Slope of the line equal to (y_target - y_ref) / (x_target - x_ref) where + ref and target x and y are any two points on the line. + :type slope: float or int + :return: Value(s) of y_target at specified x_target. + :rtype: float or int or numpy.ndarray or pandas.Series + + **Example usage** + :: + import brightwind as bw + + data = bw.load_csv(bw.demo_datasets.demo_data) + + # calculate y_target for x_target = 11 where (x_ref, y_ref) = (5, 10) is a point on the line and the slope is -0.5. + bw.utils.utils.linear_transform(x_target=11, x_ref=5, y_ref=10, slope=-0.5) + # 7.0 + + # calculate y_target for x_target as a pandas.Series and (x_ref, y_ref) = (2, 20) is a point on the line + # and the slope is -0.0065. + bw.utils.utils.linear_transform( + x_target=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065) + + # calculate y_target for x_target and x_ref as a int and y_ref as a np.ndarray where (x_ref, y_ref) = (2, y_ref) + # are points on the line and the slope is -0.0065. + bw.utils.utils.linear_transform( + x_target=20, + x_ref=2, + y_ref=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) + """ + # check input types + for var, var_name in zip([x_ref, y_ref, x_target], ['x_ref', 'y_ref', 'x_target']): + if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series) or + isinstance(var, np.ndarray)): + raise TypeError(f"{var_name} must be a float or int or a numpy.ndarray or pandas.Series.") + + if not (isinstance(slope, float) or isinstance(slope, int)): + raise TypeError("slope must be a float or int.") + + # check dimensions of x_ref and x_target if arrays or Series + if (isinstance(x_ref, np.ndarray) or isinstance(x_ref, pd.Series)) and ( + isinstance(x_target, np.ndarray) or isinstance(x_target, pd.Series)): + if len(x_ref) != len(x_target): + raise ValueError("x_ref and x_target must have the same dimensions.") + + # check dimensions of y_ref if arrays or Series + if isinstance(x_target - x_ref, np.ndarray) or isinstance(x_target - x_ref, pd.Series): + if isinstance(y_ref, np.ndarray) or isinstance(y_ref, pd.Series): + if len(y_ref) != len(x_target - x_ref): + raise ValueError("y_ref must have the same dimensions as x_target or x_ref.") + + y_target = slope * (x_target - x_ref) + y_ref + + return y_target diff --git a/tests/test_analyse.py b/tests/test_analyse.py index c1857de5..199ecc58 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -697,3 +697,48 @@ def test_scale_air_pressure_to_height(): ref_air_temp_degC=pd.Series([1, 2]), ref_height_m=2, target_height_m=10) + +def test_scale_air_density_to_height(): + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) == 1.22174 + + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, + ref_height_m=80, + target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) == 1.22 + + test_density = bw.calc_air_density(DATA.T2m, DATA.P2m) + pd.testing.assert_series_equal(bw.scale_air_density_to_height( + ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data = [1.186780, 1.187175, 1.187747, 1.185950, 1.186301 , 1.185686], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + +def test_scale_air_temperature_to_height(): + assert bw.scale_air_temperature_to_height(10.0065, 10, 11) == 10.0 + + assert bw.scale_air_temperature_to_height(ref_air_temperature=10, + ref_height_m=12, + target_height_m=10, + lapse_rate_deg_m=-0.001) == 10.002 + + pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20, + lapse_rate_deg_m=-0.001), + pd.Series(data=[0.936, 0.845, 0.713, 0.834, 0.753, 0.895], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20), + pd.Series(data=[0.837, 0.746, 0.614, 0.735, 0.654, 0.796], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) diff --git a/tests/test_utils.py b/tests/test_utils.py index 36c85a46..454a2b6c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ import pytest import brightwind as bw import pandas as pd +import numpy as np DATA = bw.load_csv(bw.demo_datasets.demo_data) DATA = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) @@ -25,3 +26,58 @@ def test_slice_data(): data_sliced = bw.utils.utils.slice_data(DATA, date_to='2017-10-23') assert data_sliced.index[0] == DATA.index[0] + +def test_linear_transform(): + # test with float and int inputs + assert bw.utils.utils.linear_transform(x_target=10, x_ref=5, y_ref=10, slope=-0.5) == 7.5 + + # test with array input for y_ref + assert (bw.utils.utils.linear_transform( + x_target=20, + x_ref=2, + y_ref=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) == np.array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796])).all() + + # test with pandas Series input for y_ref + pd.testing.assert_series_equal(bw.utils.utils.linear_transform( + x_target=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065).round(3), + pd.Series(data = [20.007, 20.007, 20.008, 20.007, 20.008, 20.007], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + # test error raising for invalid input types + with pytest.raises(TypeError): + bw.utils.utils.linear_transform( + x_target='invalid_type', + x_ref=2, + y_ref=20, + slope=-0.0065) + + #test error raising for invalid slope type + with pytest.raises(TypeError): + bw.utils.utils.linear_transform( + x_target=10, + x_ref=2, + y_ref=20, + slope='invalid_type') + + #test error raising for mismatched dimensions + with pytest.raises(ValueError): + bw.utils.utils.linear_transform( + x_target=np.array([1, 2, 3]), + x_ref=np.array([1, 2]), + y_ref=20, + slope=-0.5) + + #test error raising for mismatched dimensions with Series + with pytest.raises(ValueError): + bw.utils.utils.linear_transform( + x_target=pd.Series([1, 2, 3]), + x_ref=2, + y_ref=pd.Series([1, 2]), + slope=-0.5) + From 1ec87cf36b2176e8835371588da0ebfd32832800 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:43:45 +0100 Subject: [PATCH 14/35] [Iss535] updated calc_air_density to include humidity and follow IEC (#538) * iss #530 added lapse_temp function * iss #530 updated tests and docstring * iss #530 updated docstring * iss #530 minor docstring update * iss #531 added tests and updated function name to lapse_temperature * Revert "iss #531 added tests and updated function name to lapse_temperature" This reverts commit d41ec28e0b4f7b9e1ecf6f2ff85b8312cbe196e4. * iss #530 updated function name to lapse_temperature * iss #530 added tests * iss #534 added lapse_density function * iss #534 added tests * iss #530 updated changelog * iss #534 updated changelog * iss #530 added private function for lapsing, bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 updated name to extrapolate_temperature and updated to use bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 changed lapse_density to extrapolate_density * iss #535 updated calc_air_density to include humidity and follow IEC-recommended method * iss #530 addressed PR comments, updated function names * iss #530 small formatting fixes * iss #530 updated reference to ref for consistency with other PR * iss #530 formatting and docstring and naming fixes * iss #530 updated functions to include type for each argument * iss #535 attempt to avoid breaking change * iss 530 changed linear funct to public and fixed several typos and example issues * iss 530 fixed tests for utils and analyse * iss 530 fixed typo changelog * iss 530 updated linear_transform to raise errors and added tests * iss 530 added more tests * iss #530 added source of air density lapse rate to docstring * iss #530 include reference for air density lapse rate in docstring * iss #535 addressed comments and added workaround to avoid breaking change * iss #535 updated tests * iss #535 removed all breaking changes * iss #535 inor format change * iss #535 fix bug for dataframe input * iss #535 refactor tests * iss #535 updated tests and examples to match * iss #535 fixed bug for rel_humidity_percent * iss #535 updated constants in analyse.py * iss #535 docstring and formatting fixes * iss #535 fixed line length --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi --- CHANGELOG.md | 5 +- brightwind/analyse/analyse.py | 242 ++++++++++++++++++++++++++++------ tests/test_analyse.py | 55 ++++++-- 3 files changed, 248 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54ec8e15..66f023b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,9 @@ Additional labels for pre-release and build metadata are available as extensions ### New Features and Enhancements 1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) -1. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([[#534](https://github.com/brightwind-dev/brightwind/issues/534)]) -1. Added `scale_air_temperature_to_height` to output a air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([[#530](https://github.com/brightwind-dev/brightwind/issues/530)]) +2. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) +3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) +4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 3dbd5fb0..061b6ed2 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -42,6 +42,8 @@ # Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere GAS_CONST_DRY_AIR = 287.05 +# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) +AIR_DENSITY_LAPSE_RATE = -0.113 def dist_matrix(var_series, x_series, y_series, num_bins_x=None, num_bins_y=None, @@ -2026,64 +2028,217 @@ def sector_ratio(wspd_1, wspd_2, wdir, sectors=72, min_wspd=3, direction_bin_arr return fig -def calc_air_density(temperature, pressure, elevation_ref=None, elevation_site=None, lapse_rate=-0.113, - specific_gas_constant=286.9): +def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], + pressure: Union[float, pd.Series, pd.DataFrame], + elevation_ref: Union[float, int] = None, + elevation_site: Union[float, int] = None, + lapse_rate: float = AIR_DENSITY_LAPSE_RATE, + specific_gas_constant: float = 286.9, + rel_humidity_percent: Union[float, pd.Series, pd.DataFrame] = None + ) -> Union[float, pd.Series]: """ - Calculates air density for a given temperature and pressure and extrapolates that to the site if both reference - and site elevations are given. - - :param temperature: Temperature values in degree Celsius - :type temperature: float or pandas.Series or pandas.DataFrame - :param pressure: Pressure values in hectopascal, hPa, (1,013.25 hPa = 101,325 Pa = 101.325 kPa = - 1 atm = 1013.25 mbar) - :type pressure: float or pandas.Series or pandas.DataFrame - :param elevation_ref: Elevation, in meters, of the reference temperature and pressure location. - :type elevation_ref: Floating point value (decimal number) - :param elevation_site: Elevation, in meters, of the site location to calculate air density for. - :type elevation_site: Floating point values (decimal number) - :param lapse_rate: Air density lapse rate kg/m^3/km, default is -0.113 - :type lapse_rate: Floating point value (decimal number) - :param specific_gas_constant: Specific gas constant, R, for humid air J/(kg.K), default is 286.9 - :type specific_gas_constant: Floating point value (decimal number) - :return: Air density in kg/m^3 - :rtype: float or pandas.Series depending on the input + Calculates air density for a given air temperature (temperature), air pressure (pressure) + and relative humidity (rel_humidity_percent) using method outlined in IEC Standard (61400-12-1). + + Note that the value of the specific_gas_constant argument is used only when rel_humidity_percent is None. + This specific_gas_constant argument and its default value of 286.9 are retained to maintain backwards + compatibility for when rel_humidity_percent is None. + However, if rel_humidity_percent is not None, the specific_gas_constant is ignored as the air density + calculation which incorporates humidity depends on the two specific gas constants for dry air + (287.05 J/kg*K) and water vapour (461.5 J/kg*K) and therefore the specific_gas_constant argument is redundant. + + WARNING: The `specific_gas_constant` argument will be removed in a future 3.0 release of brightwind. + + Function gives option to scale air density to the elevation of the site if both the reference elevation + (elevation_ref) and site elevation (elevation_site) are provided. This scaling assumes that air density + decreases with increasing altitude above the earth's surface - a default air density lapse rate of -0.113 kg/m3/km + is adopted. Taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) + + WARNING: Option of scaling air density to height will be removed in a future 3.0 release of brightwind. Please call + `scale_air_density_to_height()` separately instead. + + :param temperature: Air temperature values in degree Celsius + :type temperature: float or pandas.Series or pandas.DataFrame + :param pressure: Air pressure values in hectopascal, hPa or millibar, mbar (Note 1hPa = 1mbar) + (1013.25 hPa = 101,325 Pa = 101.325 kPa = 1 atm = 1013.25 mbar) + :type pressure: float or pandas.Series or pandas.DataFrame + :param elevation_ref: Elevation, in meters, of the reference air density location (and air temperature, + air pressure, relative humidity measurements used for the calculation). + :type elevation_ref: float or int + :param elevation_site: Elevation, in meters, of the site location for which air density is calculated. + Scaled air density value(s) are calculated from elevation_ref to elevation_site + if both are specified. + :type elevation_site: float or int + :param lapse_rate: Air density lapse rate kg/m^3/km. Default is -0.113 kg/m^3/km. + :type lapse_rate: float + :param specific_gas_constant: Specific gas constant R, for moist air, in J/(kg*K). Default is 286.9 J/(kg*K). + If rel_humidity_percent is not None, this argument is ignored and the specific + gas constant for dry air of 287.05 J/(kg*K) is used instead. + If rel_humidity_percent is None, then provided the user is unconcerned with + backwards compatibility, the user should set the specific_gas_constant to 287.05 or + set rel_humidity_percent to 0 to use the specific gas constants for dry air. + :type specific_gas_constant: float + :param rel_humidity_percent: Relative humidity values as a percentage. Default is None. If None, the air density + calculation ignores humidity. + :type rel_humidity_percent: float or pandas.Series or pandas.DataFrame + :return: Air density in kg/m^3. Output type depends on type(temperature), type(pressure), + type(rel_humidity_percent) provided. If all inputs are float, output is float. + If any input is pandas.Series or pandas.DataFrame, output is pandas.Series. + :rtype: float or pandas.Series **Example usage** :: - import brightwind as bw + import brightwind as bw - #For a series of air densities - data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_site_data) - air_density = bw.calc_air_density(data.T2m, data.P2m) + data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) + + # Calculate air density from input air temperature and air pressure series + # (rel_humidity_percent=None and specific_gas_constant=286.9 by default) + bw.calc_air_density(data.T2m, data.P2m).head(5) + # Timestamp + # 2016-01-09 15:30:00 1.190011 + # 2016-01-09 15:40:00 1.190363 + # 2016-01-09 17:00:00 1.186939 + # 2016-01-09 17:10:00 1.187684 + # 2016-01-09 17:20:00 1.188079 + # dtype: float64 - #For a single value - bw.calc_air_density(15, 1013) + # Calculate air density from input air temperature and air pressure series + # (rel_humidity_percent=None and specific_gas_constant=287.05 which is the constant for dry air) + bw.calc_air_density(data.T2m, data.P2m, specific_gas_constant=287.05).head(5) + # Timestamp + # 2016-01-09 15:30:00 1.189389 + # 2016-01-09 15:40:00 1.189741 + # 2016-01-09 17:00:00 1.186319 + # 2016-01-09 17:10:00 1.187064 + # 2016-01-09 17:20:00 1.187458 + # dtype: float64 - #For a single value with ref and site elevation - bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) + # Calculate air density from single float values of air temperature and air pressure + # (rel_humidity_percent=None and specific_gas_constant=286.9 by default) + bw.calc_air_density(15, 1013).round(5) + # 1.22535 - """ + # Calculate air density from single float value of air temperature and air pressure + # (rel_humidity_percent=None and specific_gas_constant=287.05 which is the constant for dry air) + bw.calc_air_density(15, 1013, specific_gas_constant=287.05).round(5) + # 1.22471 - temp = temperature - temp_kelvin = temp + 273.15 # to convert deg C to Kelvin. - pressure = pressure * 100 # to convert hPa to Pa - ref_air_density = pressure / (specific_gas_constant * temp_kelvin) + # Calculate air density from input air temperature, air pressure and relative humidity series + # As rel_humidity_percent is not None, gas constant for dry air of 287.05 is used + bw.calc_air_density(data.T2m, data.P2m, rel_humidity_percent=data.RH2m).head(5) + # Timestamp + # 2016-01-09 15:30:00 1.186163 + # 2016-01-09 15:40:00 1.186530 + # 2016-01-09 17:00:00 1.183012 + # 2016-01-09 17:10:00 1.183790 + # 2016-01-09 17:20:00 1.184202 + # dtype: float64 + + # Calculate air density from single float value of air temperature, air pressure, relative humidity + # As rel_humidity_percent is not None, gas constant for dry air of 287.05 is used + bw.calc_air_density(15, 1013, rel_humidity_percent=50).round(5) + # 1.22093 + + # Calculate air density from a single float value of air temperature, air pressure, relative humidity + # and scale result from reference elevation to site elevation. + bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200) + # 1.198 + + """ + # Convert DataFrame inputs to Series + if isinstance(temperature, pd.DataFrame): + temperature = _convert_df_to_series(temperature).copy() + if isinstance(pressure, pd.DataFrame): + pressure = _convert_df_to_series(pressure).copy() + if isinstance(rel_humidity_percent, pd.DataFrame): + rel_humidity_percent = _convert_df_to_series(rel_humidity_percent).copy() + + # Check dimensions of temperature, pressure and rel_humidity_percent if not float or int + if isinstance(temperature, pd.Series) and (isinstance(pressure, pd.Series)): + if len(temperature) != len(pressure): + raise ValueError("temperature and pressure must have the same dimensions.") + if isinstance(rel_humidity_percent, pd.Series): + if len(temperature) != len(rel_humidity_percent): + raise ValueError("temperature, pressure and rel_humidity_percent must have the same dimensions.") + + lapse_rate_per_m = lapse_rate * 0.001 # convert lapse rate from kg/m3/km to kg/m3/m + temp_K = temperature + 273.15 # to convert deg C to Kelvin. + press_Pa = pressure * 100 # to convert hPa to Pa + vapour_press = 0.0000205 * np.exp(0.0631846 * temp_K) + specific_gas_constant_water = 461.5 + specific_gas_constant_dry_air = GAS_CONST_DRY_AIR + + if rel_humidity_percent is None: + # If relative humidity is None, the old method ignoring relative humidity is used. + # To avoid a breaking change, the value of the specific_gas_constant argument is used. + # (Default specific_gas_constant is 286.9 which is for moist air) + # If a user is not concerned with backwards compatibility, the value of this argument should be updated to match + # the specific gas constant for dry air (287.05) which makes more physical sense given relative humidity is + # taken as zero in the calculation below. + gas_constant_air = specific_gas_constant + rel_hum_decimal = 0.0 + # Deprecation warning + warnings.warn( + ( + "\nThe `specific_gas_constant` argument of `calc_air_density()` will be removed in a future 3.0 " + "release of brightwind.\nNote that this value is used only when `rel_humidity_percent` is None." + "\nPlease set `rel_humidity_percent` to 0 instead if you want to use the specific gas constant " + "for dry air (287.05 J/kg*K)." + ), + DeprecationWarning, + stacklevel=2 + ) - if elevation_ref is not None and elevation_site is not None: - site_air_density = round(ref_air_density + (((elevation_site - elevation_ref) / 1000) * lapse_rate), 3) - return site_air_density - elif elevation_site is None and elevation_ref is not None: - raise TypeError('elevation_site should be a number') - elif elevation_site is not None and elevation_ref is None: - raise TypeError('elevation_ref should be a number') else: - return ref_air_density + # Otherwise, if the rel_humidity_percent is not None (including = 0), the new method including relative humidity + # is used. The specific_gas_constant_dry_air of 287.05 is used as the air density calculation incorporating + # relative humidity is clearly split into a dry component and a moist component. Therefore it makes physical + # sense to use the specific gas constant for dry air in this context. + gas_constant_air = specific_gas_constant_dry_air + rel_hum_decimal = rel_humidity_percent * 0.01 + + # Contribution from dry air + dry_air_term = press_Pa / gas_constant_air + + # Contribution from water vapour + water_vapour_term = rel_hum_decimal * vapour_press * ( + (1 / gas_constant_air) - (1 / specific_gas_constant_water)) + + # Combine to get air density + air_density = (1 / temp_K) * (dry_air_term - water_vapour_term) + + if elevation_ref is not None and elevation_site is not None: + # Deprecation warning + warnings.warn( + ( + "\nScaling air density to height within `calc_air_density()` is deprecated and will be removed in a " + "future 3.0 release of brightwind.\nPlease call `scale_air_density_to_height()` separately instead. \n" + ), + DeprecationWarning, + stacklevel=2 + ) + # Perform extrapolation + scaled_air_density = scale_air_density_to_height(air_density, + elevation_ref, + elevation_site, + lapse_rate_per_m) + return round(scaled_air_density, 3) + + # Validate elevation arguments + if elevation_site is not None and elevation_ref is None: + raise TypeError("Specify value of elevation_ref (float or int) when elevation_site is provided.") + elif elevation_ref is not None and elevation_site is None: + raise TypeError("Specify value of elevation_site (float or int) when elevation_ref is provided.") + + return air_density def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], ref_air_temp_degC: Union[float, pd.Series], ref_height_m: Union[float, int], - target_height_m: Union[float, int]) -> Union[float, int, pd.Series]: + target_height_m: Union[float, int] + ) -> Union[float, int, pd.Series]: """ Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). @@ -2166,7 +2321,8 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], ref_height_m: float, target_height_m: float, - lapse_rate_kg_m3_m: float = -0.000113) -> Union[float, pd.Series]: + lapse_rate_kg_m3_m: float = (0.001 * AIR_DENSITY_LAPSE_RATE) + ) -> Union[float, pd.Series]: """ Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 199ecc58..a376c297 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -632,20 +632,57 @@ def test_ti_by_sector(): def test_calc_air_density(): - bw.calc_air_density(DATA[['T2m']], DATA[['P2m']]) - bw.calc_air_density(DATA.T2m, DATA.P2m) - bw.calc_air_density(DATA.T2m, DATA.P2m, elevation_ref=0, elevation_site=200) + # test DataFrame input + assert (bw.calc_air_density(DATA[['T2m']], DATA[['P2m']]).dropna().values == + bw.calc_air_density(DATA.T2m, DATA.P2m).dropna().values).all() + assert (round(bw.calc_air_density(pd.Series([15, 12.5, -5, 23]), pd.Series([1013, 990, 1020, 900])), 6 + ) == pd.Series([1.225350, 1.208010, 1.325842, 1.059254])).all() + + # test Series inputs + assert list(round(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5), 6).values + ) == [1.199177, 1.199838, 1.199794, 1.199439, 1.201066] + assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5).values - + pd.Series([1.19918, 1.19984, 1.19979, 1.19944, 1.20107])) < 1e-3).all() + assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m, rel_humidity_percent=DATA.RH2m).tail(5).values - + pd.Series([1.19529, 1.19601, 1.19592, 1.19555, 1.19719])) < 1e-3).all() + assert list(bw.calc_air_density(DATA.T2m, DATA.P2m, specific_gas_constant=287.05).head(5).round(6).dropna() + ) == [1.187064, 1.187458] + + #t test Series inputs with elevation adjustment + assert list(bw.calc_air_density(DATA.T2m, DATA.P2m, elevation_ref=0, elevation_site=200).tail(5).values + ) == [1.177, 1.177, 1.177, 1.177, 1.178] + + # test float/int inputs + assert bw.calc_air_density(15, 1013) == 1.2253503331640465 + assert (bw.calc_air_density(15, 1012, rel_humidity_percent=None, specific_gas_constant=287.05) == + bw.calc_air_density(15, 1012, rel_humidity_percent=0)) + assert round(bw.calc_air_density(15, 1012, rel_humidity_percent=0), 5) == 1.2235 + assert round(bw.calc_air_density(15, 1012), 5) == 1.22414 + assert abs(bw.calc_air_density(15, 1013, specific_gas_constant=287.05) - 1.22471) < 1e-3 + assert abs(bw.calc_air_density(15, 1013, rel_humidity_percent=50) - 1.22093) < 1e-3 + assert round(bw.calc_air_density(15, 1013, rel_humidity_percent=50, + elevation_ref=0, elevation_site=200), 5) == 1.198 + + # test float/int inputs with elevation adjustment + assert bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) == 1.203 + assert bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200 + ) - bw.scale_air_density_to_height( + bw.calc_air_density(15, 1013, rel_humidity_percent=50), 0, 200) <1e-3 + + # test errors with pytest.raises(TypeError) as except_info: bw.calc_air_density(15, 1013, elevation_site=200) - assert str(except_info.value) == 'elevation_ref should be a number' + assert str(except_info.value) == "Specify value of elevation_ref (float or int) when elevation_site is provided." with pytest.raises(TypeError) as except_info: bw.calc_air_density(15, 1013, elevation_ref=200) - assert str(except_info.value) == 'elevation_site should be a number' - assert abs(bw.calc_air_density(15, 1013) - 1.225) < 1e-3 - assert abs(bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) - 1.203) < 1e-3 - assert (abs(bw.calc_air_density(pd.Series([15, 12.5, -5, 23]), pd.Series([1013, 990, 1020, 900])) - - pd.Series([1.225, 1.208, 1.326, 1.059])) < 1e-3).all() + assert str(except_info.value) == "Specify value of elevation_site (float or int) when elevation_ref is provided." + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) + assert str(except_info.value) == "temperature and pressure must have the same dimensions." + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) + assert str(except_info.value) == "temperature, pressure and rel_humidity_percent must have the same dimensions." def test_dist_matrix_by_direction_sector(): From 9ef75ff74fab39eba3e487012185a7f2bbf106c8 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:23:43 +0100 Subject: [PATCH 15/35] [Iss541] update scale wind spd name (#542) * iss #541 renamed scale_wind_speed as apply_scale_factor, added tests and updated changelog * iss #541 updated all references to scale_wind_speed to new name * Revert "iss #541 updated all references to scale_wind_speed to new name" This reverts commit 55aa006a2e10b2c76f41eb7f8782eb2577af626a. * Revert "iss #541 renamed scale_wind_speed as apply_scale_factor, added tests and updated changelog" This reverts commit 7d2905a39d0193fb99b8e488109c23a212d2a70f. * iss #541 addressed Dan's comments, updated existing scale_wind_speed docstring to remove inaccuracy * iss #541 added apply_scale_factor to utils * iss #541 updated to allow pd.DataFrame * iss #541 updated tests and docstrings * iss #541 minor update changelog * iss #541 updates for type, wording * iss #541 minor fix * iss #541 updated changelog * iss #541 added error if series not numeric * iss #541 updated to raise value error if input is non-numeric np.ndarray * iss #541 fix PEP8 in utils * iss #541 fix PEP 8 plus some text * iss #541 add return type and fix test --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 1 + brightwind/analyse/analyse.py | 5 +-- brightwind/transform/transform.py | 59 +++++++++++++++++++++----- brightwind/utils/utils.py | 69 ++++++++++++++++++++++++++++++- tests/test_transform.py | 10 +++++ tests/test_utils.py | 10 ++++- 6 files changed, 139 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f023b9..55720797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Additional labels for pre-release and build metadata are available as extensions 2. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) 3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) 4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) +5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 061b6ed2..3844374b 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -6,7 +6,6 @@ from brightwind.utils.utils import _convert_df_to_series from brightwind.utils.utils import validate_coverage_threshold from brightwind.export.export import _calc_mean_speed_of_freq_tab -from brightwind.transform.transform import scale_wind_speed import matplotlib.pyplot as plt import warnings import textwrap @@ -1304,7 +1303,7 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 # This scale factor is used to correct `var_series` when concurrent with `direction_series`. if target_freq_table_mean is not None: scale_factor = target_freq_table_mean / data_concurrent[var_series.name].mean() - var_series_scaled = scale_wind_speed(data_concurrent[var_series.name], scale_factor) + var_series_scaled = tf.scale_wind_speed(data_concurrent[var_series.name], scale_factor) else: var_series_scaled = var_series @@ -1339,7 +1338,7 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 else: freq_tab_mean = _calc_mean_speed_of_freq_tab(result) scale_factor = target_freq_table_mean / freq_tab_mean - var_series_scaled = scale_wind_speed(var_series_scaled, scale_factor) + var_series_scaled = tf.scale_wind_speed(var_series_scaled, scale_factor) abs_percentage_diff = abs(100 * (target_freq_table_mean - freq_tab_mean) / freq_tab_mean) if abs_percentage_diff > 0.01: diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 1d301593..edcf116c 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1,12 +1,15 @@ -import datetime -import numpy as np +import copy +import datetime +import warnings +from typing import Union + +import numpy as np import pandas as pd + from brightwind.utils import utils from brightwind.load.station import _Measurements from brightwind.load.station import DATE_INSTEAD_OF_NONE from brightwind.utils.utils import validate_coverage_threshold -import copy -import warnings __all__ = ['average_data_by_period', 'merge_datasets_by_period', @@ -994,17 +997,53 @@ def apply_wspd_slope_offset_adj(data, measurements, inplace=False): return df -def scale_wind_speed(spd, scale_factor: float): +def scale_wind_speed(spd: Union[pd.Series, pd.DataFrame, float, int], + scale_factor: Union[int, float] + ) -> Union[pd.Series, pd.DataFrame, float, int]: """ Scales wind speed by the scale_factor - :param spd: Series or data frame or a single value of wind speed to scale - :param scale_factor: Scaling factor in decimal, if scaling factor is 0.8 output would be (1+0.8) times wind speed, - if it is -0.8 the output would be (1-0.8) times the wind speed - :return: Series or data frame with scaled wind speeds + :param spd: Wind speed value(s) to scale. + :type spd: pandas.Series or pandas.DataFrame or float or int + :param scale_factor: Scaling factor to use for scaling wind speed. + If scaling factor is 0.8, output would be 0.8 times wind speed. + :type scale_factor: int or float + :return: Value(s) of scaled wind speed. Output type depends on type(spd). + :rtype: pandas.Series or pandas.DataFrame or float or int + **Example usage** + :: + import brightwind as bw + import pandas as pd + import numpy as np + + data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) + + # scale float by scale_factor of 0.5 + bw.scale_wind_speed(3, 0.5) + # 1.5 + + # scale np.array by scale_factor of 0.5 + bw.scale_wind_speed(np.array([0, 1, 2]), 0.5) + # array([0. , 0.5, 1. ]) + + # scale pd.Series by scale_factor of 0.5 + bw.scale_wind_speed(data['Spd40mN'].tail(3), 0.5) + # Timestamp + # 2017-11-23 10:30:00 4.0150 + # 2017-11-23 10:40:00 3.4055 + # 2017-11-23 10:50:00 2.9325 + # Name: Spd40mN, dtype: float64 + + # scale pd.DataFrame by scale factor of 2 + bw.scale_wind_speed(data.tail(3)[['Spd60mS', 'Spd40mS']], 2) + # Spd60mS Spd40mS + # Timestamp + # 2017-11-23 10:30:00 16.900 15.750 + # 2017-11-23 10:40:00 14.318 13.336 + # 2017-11-23 10:50:00 12.808 11.498 """ - return spd * scale_factor + return utils.apply_scale_factor(spd, scale_factor) def offset_wind_direction(wdir, offset: float): diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index bd6d6361..83adf1c6 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -10,7 +10,8 @@ 'is_file', 'is_file_extension', 'validate_json', - 'linear_transform'] + 'linear_transform', + 'apply_scale_factor'] def _range_0_to_360(direction): @@ -339,3 +340,69 @@ def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], y_target = slope * (x_target - x_ref) + y_ref return y_target + + +def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], + scale_factor: Union[float, int] + ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: + """ + Scales data by the scale_factor. + + If data input is pd.DataFrame, only numeric columns are scaled. + + :param data: Data value(s) to scale by the scale_factor. + :type data: float or int or pandas.Series or pandas.DataFrame or numpy.array + :param scale_factor: Scaling factor to use for scaling data values. + :type scale_factor: float or int + :returns: Scaled data value(s). Output type depends on type(data). + :rtype: float or int or pandas.Series or pandas.DataFrame or numpy.array + + **Example usage** + :: + import brightwind as bw + import pandas as pd + import numpy as np + + # scale float by scale_factor of 0.5 + bw.utils.utils.apply_scale_factor(3, 0.5) + # 1.5 + + # # scale np.array by scale_factor of 0.5 + # bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) + # array([0. , 0.5, 1. ]) + + # scale pd.Series by scale_factor of -10 + bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) + # 0 -100 + # 1 -200 + # 2 -300 + # 3 -400 + # dtype: int64 + + # scale pd.DataFrame by scale factor of 2 + df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) + bw.utils.utils.apply_scale_factor(df, 2) + # a b c + # 0 1.0 6 a + # 1 2.4 8 b + """ + + if not isinstance(data, (float, int, pd.DataFrame, pd.Series, np.ndarray)): + raise ValueError('data should be a float or int or pd.DataFrame or pd.Series or np.ndarray') + if not isinstance(scale_factor, (float, int)): + raise ValueError('scale_factor should be a float or int') + + if isinstance(data, pd.DataFrame): + # only apply scaling to numeric columns + numeric_df = scale_factor * (data.select_dtypes(include='number')) + result = pd.concat([numeric_df, data.select_dtypes(exclude='number')], axis=1) + return result + else: + if isinstance(data, pd.Series): + if not pd.api.types.is_numeric_dtype(data): + raise ValueError('data inputted as a pandas.Series must be numeric') + if isinstance(data, np.ndarray): + if not np.issubdtype(data.dtype, np.number): + raise ValueError('data inputted as a np.ndarray must be numeric') + + return scale_factor * data diff --git a/tests/test_transform.py b/tests/test_transform.py index 7e3f176d..90a37fec 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -799,3 +799,13 @@ def test_merge_datasets_by_period(): assert round(mrgd_data['Spd80mN_Coverage'].values[0], 8) == 0.00179211 + +def test_scale_wind_speed(): + + assert bw.scale_wind_speed(3, 0.5) == 1.5 + assert (bw.scale_wind_speed(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() + assert (bw.scale_wind_speed(DATA_CLND['Spd40mN'].tail(3), 0.5)==[4.015 , 3.4055, 2.9325]).all() + + df = pd.DataFrame({'Spd_100m':[0.5, 1.2], 'Spd_100m':[3, 4], 'c':['a', 'b']}) + result_df = pd.DataFrame({'Spd_100m':[1.0, 2.4], 'Spd_100m':[6, 8], 'c':['a', 'b']}) + assert result_df.equals(bw.scale_wind_speed(df, 2)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 454a2b6c..a188573f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,9 @@ import pytest -import brightwind as bw import pandas as pd import numpy as np +import brightwind as bw + DATA = bw.load_csv(bw.demo_datasets.demo_data) DATA = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) WSPD_COLS = ['Spd80mN', 'Spd80mS', 'Spd60mN', 'Spd60mS', 'Spd40mN', 'Spd40mS'] @@ -81,3 +82,10 @@ def test_linear_transform(): y_ref=pd.Series([1, 2]), slope=-0.5) +def test_apply_scale_factor(): + assert bw.utils.utils.apply_scale_factor(3, 0.5) == 1.5 + assert (bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() + assert (bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) == [-100, -200, -300, -400]).all() + df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) + result_df = pd.DataFrame({'a':[1.0, 2.4], 'b':[6, 8], 'c':['a', 'b']}) + assert result_df.equals(bw.utils.utils.apply_scale_factor(df, 2)) From b270f3d7aed43cb1155ca48fd6d65c2b37353ea6 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:48:45 +0000 Subject: [PATCH 16/35] Moved scaling functions to transform/scale.py (SR1) (#548) * iss #544 moved scaling functions to transform/scale.py * iss #544 updated paths in examples * iss #544 reordered functions * iss #544 PEP8 formatting * iss #544 fix indent error + PEP8 * iss #544 fixed flake8 issues in scale.py and test_transform.py * iss #544 fixed flake8 in test_scale.py --------- Co-authored-by: stephenholleran --- brightwind/__init__.py | 1 + brightwind/analyse/analyse.py | 228 +---------------- brightwind/transform/scale.py | 390 ++++++++++++++++++++++++++++++ brightwind/transform/transform.py | 3 +- brightwind/utils/utils.py | 152 +----------- tests/test_analyse.py | 86 +------ tests/test_scale.py | 160 ++++++++++++ tests/test_transform.py | 8 +- tests/test_utils.py | 63 ----- 9 files changed, 565 insertions(+), 526 deletions(-) create mode 100644 brightwind/transform/scale.py create mode 100644 tests/test_scale.py diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 20152832..5814a0f6 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -5,6 +5,7 @@ from .analyse.analyse import * from .analyse.plot import * from .transform.transform import * +from .transform.scale import * from .export.export import * from . import demo_datasets from .utils.gis import * diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 3844374b..0c5d746c 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -3,6 +3,7 @@ from brightwind.transform import transform as tf from brightwind.utils import utils from brightwind.analyse import plot as bw_plt +from brightwind.transform.scale import scale_air_density_to_height from brightwind.utils.utils import _convert_df_to_series from brightwind.utils.utils import validate_coverage_threshold from brightwind.export.export import _calc_mean_speed_of_freq_tab @@ -27,16 +28,7 @@ 'basic_stats', 'TI', 'sector_ratio', - 'calc_air_density', - 'scale_air_pressure_to_height', - 'scale_air_density_to_height', - 'scale_air_temperature_to_height'] - -# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere -ACCEL_DUE_TO_GRAVITY = 9.80665 - -# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere -TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + 'calc_air_density'] # Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere GAS_CONST_DRY_AIR = 287.05 @@ -2219,9 +2211,9 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], ) # Perform extrapolation scaled_air_density = scale_air_density_to_height(air_density, - elevation_ref, - elevation_site, - lapse_rate_per_m) + elevation_ref, + elevation_site, + lapse_rate_per_m) return round(scaled_air_density, 3) # Validate elevation arguments @@ -2233,213 +2225,3 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], return air_density -def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], - ref_air_temp_degC: Union[float, pd.Series], - ref_height_m: Union[float, int], - target_height_m: Union[float, int] - ) -> Union[float, int, pd.Series]: - """ - Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) - and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). - - Calculation based on ISO:2533-1975 Standard Atmosphere (https://www.iso.org/obp/ui/#iso:std:iso:2533:en) - as suggested by IEC 61400-12-1: - - scaled_air_pressure = ref_air_pressure_hPa*((1 + (L/ref_air_temp_K)*(target_height_m - ref_height_m))**(-g/(L*R))) - where: - g = 9.80665 is acceleration due to gravity (m/s^2) - L = -0.0065 is the temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) - R = 287.05 is the specific gas constant for dry air (J/K/kg or m2/K/s2) - - :param ref_air_pressure_hPa: Reference air pressure value(s) in hPa (1mbar = 1hPa = 100Pa). - :type ref_air_pressure_hPa: float or int or pandas.Series - :param ref_air_temp_degC: Reference air temperature value(s) in degrees celsius. - :type ref_air_temp_degC: float or int or pandas.Series - :param ref_height_m: Height (in metres) of reference air temperature (ref_air_temp_degC) - and air pressure (ref_air_pressure_hPa). - :type ref_height_m: float or int - :param target_height_m: Height (in metres) which ref_air_pressure_hPa is scaled to. - :type target_height_m float or int - :return: Air pressure at specified height of target_height_m in hPa (1mbar = 1hPa = 100Pa). - Type depends on type(ref_air_pressure_hPa) and type(ref_air_temp_degC) inputs. - :rtype: float or int or pandas.Series - - **Example usage** - :: - - import brightwind as bw - - # scale float value of air pressure - round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, ref_height_m=10, - target_height_m=200), 2) - # 977.45 - - # scale air pressure based on input series of reference air pressure and air temperature - data = bw.load_csv(bw.demo_datasets.demo_data) - - bw.scale_air_pressure_to_height(ref_air_pressure_hPa=data['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_air_temp_degC=data['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10).round(2) - # Timestamp - # 2016-01-09 17:10:00 933.07 - # 2016-01-09 17:20:00 933.07 - # 2016-01-09 17:30:00 933.07 - # 2016-01-09 17:40:00 932.07 - # 2016-01-09 17:50:00 932.07 - # 2016-01-09 18:00:00 932.07 - # dtype: float64 - - """ - - # check input types - for var, var_name in zip([ref_air_pressure_hPa, ref_air_temp_degC], ['ref_air_pressure_hPa', 'ref_air_temp_degC']): - if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series)): - raise TypeError(f"{var_name} must be a float or int or pandas.Series.") - for var, var_name in zip([ref_height_m, target_height_m], ['ref_height_m', 'target_height_m']): - if not (isinstance(var, float) or isinstance(var, int)): - raise TypeError(f"{var_name} must be a float or int.") - - # check dimensions of ref_air_pressure_hPa and ref_air_temp_degC if Series - if isinstance(ref_air_pressure_hPa, pd.Series) and (isinstance(ref_air_temp_degC, pd.Series)): - if len(ref_air_pressure_hPa) != len(ref_air_temp_degC): - raise ValueError("ref_air_pressure_hPa and ref_air_temp_degC must have the same dimensions.") - - # Constants as outlined in ISO:2533 - g = ACCEL_DUE_TO_GRAVITY # Acceleration due to gravity (m/s^2) - L = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE # Temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) - R = GAS_CONST_DRY_AIR # Specific gas constant dry air - - ref_air_temp_K = ref_air_temp_degC + 273.15 # Convert temp units to K - - scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) - ) ** (-g / (L * R))) - - return scaled_air_pressure_hPa - - -def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], - ref_height_m: float, - target_height_m: float, - lapse_rate_kg_m3_m: float = (0.001 * AIR_DENSITY_LAPSE_RATE) - ) -> Union[float, pd.Series]: - """ - Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height - (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. - - :param ref_air_density_kg_m3: Reference air density value(s) in kg/m3. - :type ref_air_density_kg_m3: float or pandas.Series - :param ref_height_m: Measurement height (in metres) of ref_air_density_kg_m3. - :type ref_height_m float - :param target_height_m: Height (in metres) that ref_air_density_kg_m3 is scaled to. - :type target_height_m: float - :param lapse_rate_kg_m3_m: Lapse rate describes how air density changes with increasing height above the - earth's surface in kg/m3/m. - Default value of -0.113 kg/m3 per km above earth's surface (-0.000113 kg/m3/m) - taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014). - :type lapse_rate_kg_m3_m: float - :return: Air density at specified height of target_height_m in kg/m3. Type depends on - type(ref_air_density_kg_m3) input. - :rtype: float or pandas.Series - - **Example usage** - :: - import brightwind as bw - - # scale float value of air density using default lapse_rate_kg_m3_m - bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) - # 1.22174 - - # scale float value of air density using non-default value for lapse_rate_kg_m3_m - bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100, - lapse_rate_kg_m3_m=-0.0002) - # 1.22 - - # derive air density and scale based on series input values for reference air density - data = bw.load_csv(bw.demo_datasets.demo_data) - test_density = bw.calc_air_density(data.T2m, data.P2m) - bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10) - - Timestamp - 2016-01-09 17:10:00 1.186780 - 2016-01-09 17:20:00 1.187175 - 2016-01-09 17:30:00 1.187747 - 2016-01-09 17:40:00 1.185950 - 2016-01-09 17:50:00 1.186301 - 2016-01-09 18:00:00 1.185686 - dtype: float64 - """ - - scaled_air_density = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, - y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) - - return scaled_air_density - - -def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series], - ref_height_m: float, - target_height_m: float, - lapse_rate_deg_m: float = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE - ) -> Union[float, pd.Series]: - """ - Linearly scales reference air temperature measurement (ref_air_temperature) from its measurement height - (ref_height_m) to the height specified as target_height_m, by applying the constant lapse_rate_deg_m. - - :param ref_air_temperature: Air temperature value(s) in degrees [for example in Celsius or Kelvin]. - :type ref_air_temperature: float or pandas.Series - :param ref_height_m: Measurement height (in metres) of ref_air_temperature. - :type ref_height_m: float - :param target_height_m: Height (in metres) that ref_air_temperature is scaled to. - :type target_height_m: float - :param lapse_rate_deg_m: Lapse rate describes how air temperature changes with increasing height - above the earth's surface. - Units should be degrees of temperature per unit of height, e.g. °C/m or K/m. - Default value of -6.5 degrees Celsius per km above the earth's surface - (or -0.0065 °C/m) is commonly used as an approximation of the - atmospheric lapse rate. - In particular, the IEC standards rely on the ISO2533:1975 Standard Atmosphere - which states that a lapse rate of 6.5 K/km is valid for geopotential altitudes - of up to 11 km above earth's surface. - This value was also adopted in WASP 11: - Mortensen, N. G., Heathfield, D. N., Rathmann, O., & Nielsen, M. (2014). - Wind Atlas Analysis and Application Program: WAsP 11 Help Facility. Computer - programme, Department of Wind Energy, Technical University of Denmark - https://orbit.dtu.dk/en/publications/wind-atlas-analysis-and-application-program-wasp-11-help-facility - :type lapse_rate_deg_m: float - :return: Air temperature at specified height of target_height_m in same unit as input - ref_air_temperature [for example in Celsius or Kelvin]. Output type depends on - type(ref_air_temperature) input. - :rtype: float or pandas.Series - - **Example usage** - :: - - import brightwind as bw - - # scale air temperature based on float input value for reference air temperature - bw.scale_air_temperature_to_height(ref_air_temperature=10.0065, ref_height_m=10, target_height_m=11) - # 10.0 - - # scale air temperature based on float input value for reference air temperature with non-default lapse_rate_deg_m - bw.scale_air_temperature_to_height(ref_air_temperature=10, ref_height_m=12, target_height_m=10, - lapse_rate_deg_m=-0.001) - # 10.002 - - # scale air temperature based on series input values for reference air temperature - data = bw.load_csv(bw.demo_datasets.demo_data) - - bw.scale_air_temperature_to_height(ref_air_temperature=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=20) - # Timestamp - # 2016-01-09 17:10:00 0.837 - # 2016-01-09 17:20:00 0.746 - # 2016-01-09 17:30:00 0.614 - # 2016-01-09 17:40:00 0.735 - # 2016-01-09 17:50:00 0.654 - # 2016-01-09 18:00:00 0.796 - # Name: T2m, dtype: float64 - """ - - scaled_air_temp = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, - y_ref=ref_air_temperature, slope=lapse_rate_deg_m) - return scaled_air_temp diff --git a/brightwind/transform/scale.py b/brightwind/transform/scale.py new file mode 100644 index 00000000..7bf903ea --- /dev/null +++ b/brightwind/transform/scale.py @@ -0,0 +1,390 @@ +from typing import Union +import numpy as np +import pandas as pd + +__all__ = ['apply_scale_factor', + 'linear_transform', + 'scale_air_density_to_height', + 'scale_air_temperature_to_height', + 'scale_air_pressure_to_height'] + +# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere +ACCEL_DUE_TO_GRAVITY = 9.80665 + +# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere +TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + +# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere +GAS_CONST_DRY_AIR = 287.05 + +# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) +AIR_DENSITY_LAPSE_RATE = -0.113 + + +def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], + scale_factor: Union[float, int] + ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: + """ + Scales data by the scale_factor. + + If data input is pd.DataFrame, only numeric columns are scaled. + + :param data: Data value(s) to scale by the scale_factor. + :type data: float or int or pandas.Series or pandas.DataFrame or numpy.array + :param scale_factor: Scaling factor to use for scaling data values. + :type scale_factor: float or int + :returns: Scaled data value(s). Output type depends on type(data). + :rtype: float or int or pandas.Series or pandas.DataFrame or numpy.array + + **Example usage** + :: + import brightwind as bw + import pandas as pd + import numpy as np + + # scale float by scale_factor of 0.5 + bw.apply_scale_factor(3, 0.5) + # 1.5 + + # # scale np.array by scale_factor of 0.5 + bw.apply_scale_factor(np.array([0, 1, 2]), 0.5) + # array([0. , 0.5, 1. ]) + + # scale pd.Series by scale_factor of -10 + bw.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) + # 0 -100 + # 1 -200 + # 2 -300 + # 3 -400 + # dtype: int64 + + # scale pd.DataFrame by scale factor of 2 + df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) + bw.apply_scale_factor(df, 2) + # a b c + # 0 1.0 6 a + # 1 2.4 8 b + """ + + if not isinstance(data, (float, int, pd.DataFrame, pd.Series, np.ndarray)): + raise ValueError('data should be a float or int or pd.DataFrame or pd.Series or np.ndarray') + if not isinstance(scale_factor, (float, int)): + raise ValueError('scale_factor should be a float or int') + + if isinstance(data, pd.DataFrame): + # only apply scaling to numeric columns + numeric_df = scale_factor * (data.select_dtypes(include='number')) + result = pd.concat([numeric_df, data.select_dtypes(exclude='number')], axis=1) + return result + if isinstance(data, pd.Series): + if not pd.api.types.is_numeric_dtype(data): + raise ValueError('data inputted as a pandas.Series must be numeric') + if isinstance(data, np.ndarray): + if not np.issubdtype(data.dtype, np.number): + raise ValueError('data inputted as a np.ndarray must be numeric') + return scale_factor * data + + +def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], + x_ref: Union[float, int, np.ndarray, pd.Series], + y_ref: Union[float, int, np.ndarray, pd.Series], + slope: Union[float, int]) -> Union[float, int, np.ndarray, pd.Series]: + """ + Perform a linear transform of known (x_ref, y_ref) to calculate y_target, + using a constant slope and known value(s) of x_target. + + Function applies a linear transformation based on the equation of a straight line with constant slope: + y_target = slope * (x_target - x_ref) + y_ref, + where (x_ref, y_ref) is effectively considered a known point on a line with the inputted slope. + + Note that if pd.Series or np.ndarray inputs are provided for x_target, x_ref and y_ref, + then the transform is performed on a point by point basis, as in, the n^{th} value of + the resulting y_target would be: + y_target[n] = slope * (x_target[n] - x_ref[n]) + y_ref[n], + where (x_ref[n], y_ref[n]) is effectively considered a known point on a line with the inputted slope. + Therefore all three of x_target, x_ref and y_ref must have the same dimensions. + The inputted slope is used even in the case of pd.Series or np.ndarray inputs as the purpose of this function + is to perform a linear transformation using a predefined slope, not to fit a line to the input data. + + :param x_target: Target x value(s) at which to calculate y_target. + :type x_target: float or int or numpy.ndarray or pandas.Series + :param x_ref: Reference x value(s) of known reference point(s) on the line. + :type x_ref: float or int or numpy.ndarray or pandas.Series + :param y_ref: Reference y value(s) of known reference point(s) on the line. + :type y_ref: float or int or numpy.ndarray or pandas.Series + :param slope: Slope of the line equal to (y_target - y_ref) / (x_target - x_ref) where + ref and target x and y are any two points on the line. + :type slope: float or int + :return: Value(s) of y_target at specified x_target. + :rtype: float or int or numpy.ndarray or pandas.Series + + **Example usage** + :: + import brightwind as bw + + data = bw.load_csv(bw.demo_datasets.demo_data) + + # calculate y_target for x_target = 11 where (x_ref, y_ref) = (5, 10) is a point on the line and the slope is -0.5. + bw.linear_transform(x_target=11, x_ref=5, y_ref=10, slope=-0.5) + # 7.0 + + # calculate y_target for x_target as a pandas.Series and (x_ref, y_ref) = (2, 20) is a point on the line + # and the slope is -0.0065. + bw.linear_transform( + x_target=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065) + # Timestamp + # 2016-01-09 17:10:00 20.006799 + # 2016-01-09 17:20:00 20.007390 + # 2016-01-09 17:30:00 20.008249 + # 2016-01-09 17:40:00 20.007462 + # 2016-01-09 17:50:00 20.007988 + # 2016-01-09 18:00:00 20.007065 + # Name: T2m, dtype: float64 + + # calculate y_target for x_target and x_ref as a int and y_ref as a np.ndarray where (x_ref, y_ref) = (2, y_ref) + # are points on the line and the slope is -0.0065. + bw.linear_transform( + x_target=20, + x_ref=2, + y_ref=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) + # array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796]) + """ + # check input types + for var, var_name in zip([x_ref, y_ref, x_target], ['x_ref', 'y_ref', 'x_target']): + if not isinstance(var, (float, int, pd.Series, np.ndarray)): + raise TypeError(f"{var_name} must be a float or int or a numpy.ndarray or pandas.Series.") + + if not isinstance(slope, (float, int)): + raise TypeError("slope must be a float or int.") + + # check dimensions of x_ref and x_target if arrays or Series + if (isinstance(x_ref, (np.ndarray, pd.Series))) and ( + isinstance(x_target, (np.ndarray, pd.Series))): + if len(x_ref) != len(x_target): + raise ValueError("x_ref and x_target must have the same dimensions.") + + # check dimensions of y_ref if arrays or Series + if isinstance(x_target - x_ref, (np.ndarray, pd.Series)): + if isinstance(y_ref, (np.ndarray, pd.Series)): + if len(y_ref) != len(x_target - x_ref): + raise ValueError("y_ref must have the same dimensions as x_target or x_ref.") + + y_target = slope * (x_target - x_ref) + y_ref + + return y_target + + +def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_kg_m3_m: float = (0.001 * AIR_DENSITY_LAPSE_RATE) + ) -> Union[float, pd.Series]: + """ + Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height + (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. + + :param ref_air_density_kg_m3: Reference air density value(s) in kg/m3. + :type ref_air_density_kg_m3: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_density_kg_m3. + :type ref_height_m float + :param target_height_m: Height (in metres) that ref_air_density_kg_m3 is scaled to. + :type target_height_m: float + :param lapse_rate_kg_m3_m: Lapse rate describes how air density changes with increasing height above the + earth's surface in kg/m3/m. + Default value of -0.113 kg/m3 per km above earth's surface (-0.000113 kg/m3/m) + taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014). + :type lapse_rate_kg_m3_m: float + :return: Air density at specified height of target_height_m in kg/m3. Type depends on + type(ref_air_density_kg_m3) input. + :rtype: float or pandas.Series + + **Example usage** + :: + import brightwind as bw + + # scale float value of air density using default lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) + # 1.22174 + + # scale float value of air density using non-default value for lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) + # 1.22 + + # derive air density and scale based on series input values for reference air density + data = bw.load_csv(bw.demo_datasets.demo_data) + test_density = bw.calc_air_density(data.T2m, data.P2m) + bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10) + + Timestamp + 2016-01-09 17:10:00 1.186780 + 2016-01-09 17:20:00 1.187175 + 2016-01-09 17:30:00 1.187747 + 2016-01-09 17:40:00 1.185950 + 2016-01-09 17:50:00 1.186301 + 2016-01-09 18:00:00 1.185686 + dtype: float64 + """ + + scaled_air_density = linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) + + return scaled_air_density + + +def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_deg_m: float = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE + ) -> Union[float, pd.Series]: + """ + Linearly scales reference air temperature measurement (ref_air_temperature) from its measurement height + (ref_height_m) to the height specified as target_height_m, by applying the constant lapse_rate_deg_m. + + :param ref_air_temperature: Air temperature value(s) in degrees [for example in Celsius or Kelvin]. + :type ref_air_temperature: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_temperature. + :type ref_height_m: float + :param target_height_m: Height (in metres) that ref_air_temperature is scaled to. + :type target_height_m: float + :param lapse_rate_deg_m: Lapse rate describes how air temperature changes with increasing height + above the earth's surface. + Units should be degrees of temperature per unit of height, e.g. °C/m or K/m. + Default value of -6.5 degrees Celsius per km above the earth's surface + (or -0.0065 °C/m) is commonly used as an approximation of the + atmospheric lapse rate. + In particular, the IEC standards rely on the ISO2533:1975 Standard Atmosphere + which states that a lapse rate of 6.5 K/km is valid for geopotential altitudes + of up to 11 km above earth's surface. + This value was also adopted in WASP 11: + Mortensen, N. G., Heathfield, D. N., Rathmann, O., & Nielsen, M. (2014). + Wind Atlas Analysis and Application Program: WAsP 11 Help Facility. Computer + programme, Department of Wind Energy, Technical University of Denmark + https://orbit.dtu.dk/en/publications/wind-atlas-analysis-and-application-program-wasp-11-help-facility + :type lapse_rate_deg_m: float + :return: Air temperature at specified height of target_height_m in same unit as input + ref_air_temperature [for example in Celsius or Kelvin]. Output type depends on + type(ref_air_temperature) input. + :rtype: float or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale air temperature based on float input value for reference air temperature + bw.scale_air_temperature_to_height(ref_air_temperature=10.0065, ref_height_m=10, target_height_m=11) + # 10.0 + + # scale air temperature based on float input value for reference air temperature with non-default lapse_rate_deg_m + bw.scale_air_temperature_to_height(ref_air_temperature=10, ref_height_m=12, target_height_m=10, + lapse_rate_deg_m=-0.001) + # 10.002 + + # scale air temperature based on series input values for reference air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_temperature_to_height(ref_air_temperature=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=20) + # Timestamp + # 2016-01-09 17:10:00 0.837 + # 2016-01-09 17:20:00 0.746 + # 2016-01-09 17:30:00 0.614 + # 2016-01-09 17:40:00 0.735 + # 2016-01-09 17:50:00 0.654 + # 2016-01-09 18:00:00 0.796 + # Name: T2m, dtype: float64 + """ + + scaled_air_temp = linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_temperature, slope=lapse_rate_deg_m) + return scaled_air_temp + + +def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], + ref_air_temp_degC: Union[float, pd.Series], + ref_height_m: Union[float, int], + target_height_m: Union[float, int] + ) -> Union[float, int, pd.Series]: + """ + Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) + and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). + + Calculation based on ISO:2533-1975 Standard Atmosphere (https://www.iso.org/obp/ui/#iso:std:iso:2533:en) + as suggested by IEC 61400-12-1: + + scaled_air_pressure = ref_air_pressure_hPa*((1 + (L/ref_air_temp_K)*(target_height_m - ref_height_m))**(-g/(L*R))) + where: + g = 9.80665 is acceleration due to gravity (m/s^2) + L = -0.0065 is the temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = 287.05 is the specific gas constant for dry air (J/K/kg or m2/K/s2) + + :param ref_air_pressure_hPa: Reference air pressure value(s) in hPa (1mbar = 1hPa = 100Pa). + :type ref_air_pressure_hPa: float or int or pandas.Series + :param ref_air_temp_degC: Reference air temperature value(s) in degrees celsius. + :type ref_air_temp_degC: float or int or pandas.Series + :param ref_height_m: Height (in metres) of reference air temperature (ref_air_temp_degC) + and air pressure (ref_air_pressure_hPa). + :type ref_height_m: float or int + :param target_height_m: Height (in metres) which ref_air_pressure_hPa is scaled to. + :type target_height_m float or int + :return: Air pressure at specified height of target_height_m in hPa (1mbar = 1hPa = 100Pa). + Type depends on type(ref_air_pressure_hPa) and type(ref_air_temp_degC) inputs. + :rtype: float or int or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale float value of air pressure + round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, ref_height_m=10, + target_height_m=200), 2) + # 977.45 + + # scale air pressure based on input series of reference air pressure and air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_pressure_to_height(ref_air_pressure_hPa=data['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=data['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10).round(2) + # Timestamp + # 2016-01-09 17:10:00 933.07 + # 2016-01-09 17:20:00 933.07 + # 2016-01-09 17:30:00 933.07 + # 2016-01-09 17:40:00 932.07 + # 2016-01-09 17:50:00 932.07 + # 2016-01-09 18:00:00 932.07 + # dtype: float64 + + """ + + # check input types + for var, var_name in zip([ref_air_pressure_hPa, ref_air_temp_degC], ['ref_air_pressure_hPa', 'ref_air_temp_degC']): + if not isinstance(var, (float, int, pd.Series)): + raise TypeError(f"{var_name} must be a float or int or pandas.Series.") + for var, var_name in zip([ref_height_m, target_height_m], ['ref_height_m', 'target_height_m']): + if not isinstance(var, (float, int)): + raise TypeError(f"{var_name} must be a float or int.") + + # check dimensions of ref_air_pressure_hPa and ref_air_temp_degC if Series + if isinstance(ref_air_pressure_hPa, pd.Series) and (isinstance(ref_air_temp_degC, pd.Series)): + if len(ref_air_pressure_hPa) != len(ref_air_temp_degC): + raise ValueError("ref_air_pressure_hPa and ref_air_temp_degC must have the same dimensions.") + + # Constants as outlined in ISO:2533 + g = ACCEL_DUE_TO_GRAVITY # Acceleration due to gravity (m/s^2) + L = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE # Temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = GAS_CONST_DRY_AIR # Specific gas constant dry air + + ref_air_temp_K = ref_air_temp_degC + 273.15 # Convert temp units to K + + scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) + ) ** (-g / (L * R))) + + return scaled_air_pressure_hPa diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index edcf116c..509f687c 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +import brightwind.transform.scale from brightwind.utils import utils from brightwind.load.station import _Measurements from brightwind.load.station import DATE_INSTEAD_OF_NONE @@ -1043,7 +1044,7 @@ def scale_wind_speed(spd: Union[pd.Series, pd.DataFrame, float, int], # 2017-11-23 10:40:00 14.318 13.336 # 2017-11-23 10:50:00 12.808 11.498 """ - return utils.apply_scale_factor(spd, scale_factor) + return brightwind.transform.scale.apply_scale_factor(spd, scale_factor) def offset_wind_direction(wdir, offset: float): diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index 83adf1c6..2c56188a 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -9,9 +9,8 @@ 'validate_coverage_threshold', 'is_file', 'is_file_extension', - 'validate_json', - 'linear_transform', - 'apply_scale_factor'] + 'validate_json' + ] def _range_0_to_360(direction): @@ -257,152 +256,5 @@ def validate_json(json_to_check, schema): return data_is_valid -def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], - x_ref: Union[float, int, np.ndarray, pd.Series], - y_ref: Union[float, int, np.ndarray, pd.Series], - slope: Union[float, int]) -> Union[float, int, np.ndarray, pd.Series]: - """ - Perform a linear transform of known (x_ref, y_ref) to calculate y_target, - using a constant slope and known value(s) of x_target. - - Function applies a linear transformation based on the equation of a straight line with constant slope: - y_target = slope * (x_target - x_ref) + y_ref, - where (x_ref, y_ref) is effectively considered a known point on a line with the inputted slope. - - Note that if pd.Series or np.ndarray inputs are provided for x_target, x_ref and y_ref, - then the transform is performed on a point by point basis, as in, the n^{th} value of - the resulting y_target would be: - y_target[n] = slope * (x_target[n] - x_ref[n]) + y_ref[n], - where (x_ref[n], y_ref[n]) is effectively considered a known point on a line with the inputted slope. - Therefore all three of x_target, x_ref and y_ref must have the same dimensions. - The inputted slope is used even in the case of pd.Series or np.ndarray inputs as the purpose of this function - is to perform a linear transformation using a predefined slope, not to fit a line to the input data. - - :param x_target: Target x value(s) at which to calculate y_target. - :type x_target: float or int or numpy.ndarray or pandas.Series - :param x_ref: Reference x value(s) of known reference point(s) on the line. - :type x_ref: float or int or numpy.ndarray or pandas.Series - :param y_ref: Reference y value(s) of known reference point(s) on the line. - :type y_ref: float or int or numpy.ndarray or pandas.Series - :param slope: Slope of the line equal to (y_target - y_ref) / (x_target - x_ref) where - ref and target x and y are any two points on the line. - :type slope: float or int - :return: Value(s) of y_target at specified x_target. - :rtype: float or int or numpy.ndarray or pandas.Series - - **Example usage** - :: - import brightwind as bw - - data = bw.load_csv(bw.demo_datasets.demo_data) - - # calculate y_target for x_target = 11 where (x_ref, y_ref) = (5, 10) is a point on the line and the slope is -0.5. - bw.utils.utils.linear_transform(x_target=11, x_ref=5, y_ref=10, slope=-0.5) - # 7.0 - - # calculate y_target for x_target as a pandas.Series and (x_ref, y_ref) = (2, 20) is a point on the line - # and the slope is -0.0065. - bw.utils.utils.linear_transform( - x_target=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - x_ref=2, - y_ref=20, - slope=-0.0065) - - # calculate y_target for x_target and x_ref as a int and y_ref as a np.ndarray where (x_ref, y_ref) = (2, y_ref) - # are points on the line and the slope is -0.0065. - bw.utils.utils.linear_transform( - x_target=20, - x_ref=2, - y_ref=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, - slope=-0.0065) - """ - # check input types - for var, var_name in zip([x_ref, y_ref, x_target], ['x_ref', 'y_ref', 'x_target']): - if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series) or - isinstance(var, np.ndarray)): - raise TypeError(f"{var_name} must be a float or int or a numpy.ndarray or pandas.Series.") - - if not (isinstance(slope, float) or isinstance(slope, int)): - raise TypeError("slope must be a float or int.") - - # check dimensions of x_ref and x_target if arrays or Series - if (isinstance(x_ref, np.ndarray) or isinstance(x_ref, pd.Series)) and ( - isinstance(x_target, np.ndarray) or isinstance(x_target, pd.Series)): - if len(x_ref) != len(x_target): - raise ValueError("x_ref and x_target must have the same dimensions.") - - # check dimensions of y_ref if arrays or Series - if isinstance(x_target - x_ref, np.ndarray) or isinstance(x_target - x_ref, pd.Series): - if isinstance(y_ref, np.ndarray) or isinstance(y_ref, pd.Series): - if len(y_ref) != len(x_target - x_ref): - raise ValueError("y_ref must have the same dimensions as x_target or x_ref.") - - y_target = slope * (x_target - x_ref) + y_ref - - return y_target -def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], - scale_factor: Union[float, int] - ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: - """ - Scales data by the scale_factor. - - If data input is pd.DataFrame, only numeric columns are scaled. - - :param data: Data value(s) to scale by the scale_factor. - :type data: float or int or pandas.Series or pandas.DataFrame or numpy.array - :param scale_factor: Scaling factor to use for scaling data values. - :type scale_factor: float or int - :returns: Scaled data value(s). Output type depends on type(data). - :rtype: float or int or pandas.Series or pandas.DataFrame or numpy.array - - **Example usage** - :: - import brightwind as bw - import pandas as pd - import numpy as np - - # scale float by scale_factor of 0.5 - bw.utils.utils.apply_scale_factor(3, 0.5) - # 1.5 - - # # scale np.array by scale_factor of 0.5 - # bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) - # array([0. , 0.5, 1. ]) - - # scale pd.Series by scale_factor of -10 - bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) - # 0 -100 - # 1 -200 - # 2 -300 - # 3 -400 - # dtype: int64 - - # scale pd.DataFrame by scale factor of 2 - df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) - bw.utils.utils.apply_scale_factor(df, 2) - # a b c - # 0 1.0 6 a - # 1 2.4 8 b - """ - - if not isinstance(data, (float, int, pd.DataFrame, pd.Series, np.ndarray)): - raise ValueError('data should be a float or int or pd.DataFrame or pd.Series or np.ndarray') - if not isinstance(scale_factor, (float, int)): - raise ValueError('scale_factor should be a float or int') - - if isinstance(data, pd.DataFrame): - # only apply scaling to numeric columns - numeric_df = scale_factor * (data.select_dtypes(include='number')) - result = pd.concat([numeric_df, data.select_dtypes(exclude='number')], axis=1) - return result - else: - if isinstance(data, pd.Series): - if not pd.api.types.is_numeric_dtype(data): - raise ValueError('data inputted as a pandas.Series must be numeric') - if isinstance(data, np.ndarray): - if not np.issubdtype(data.dtype, np.number): - raise ValueError('data inputted as a np.ndarray must be numeric') - - return scale_factor * data diff --git a/tests/test_analyse.py b/tests/test_analyse.py index a376c297..2a3cf01a 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -667,7 +667,7 @@ def test_calc_air_density(): # test float/int inputs with elevation adjustment assert bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) == 1.203 assert bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200 - ) - bw.scale_air_density_to_height( + ) - bw.transform.scale.scale_air_density_to_height( bw.calc_air_density(15, 1013, rel_humidity_percent=50), 0, 200) <1e-3 # test errors @@ -695,87 +695,3 @@ def test_dist_matrix_by_direction_sector(): bw.dist_matrix_by_dir_sector(DATA.Spd40mN, DATA.T2m, DATA.Dir38mS, var_to_bin_by_array=[-8, -5, 5, 10, 15, 20, 26], sectors=8) assert True - - -def test_scale_air_pressure_to_height(): - assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, - ref_height_m=10, target_height_m=200), 2)== 977.45 - assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, - ref_height_m=10, target_height_m=15), 2) == 999.4 - pd.testing.assert_series_equal(bw.scale_air_pressure_to_height( - ref_air_pressure_hPa=DATA['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_air_temp_degC=DATA['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10), - pd.Series(data = [933.07, 933.07, 933.07, 932.07, 932.07, 932.07], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - # test error raising for invalid input types - with pytest.raises(TypeError): - bw.scale_air_pressure_to_height( - ref_air_pressure_hPa='invalid_type', - ref_air_temp_degC=12, - ref_height_m=2, - target_height_m=10) - - # test error raising for invalid slope type - with pytest.raises(TypeError): - bw.scale_air_pressure_to_height( - ref_air_pressure_hPa=1000, - ref_air_temp_degC=12, - ref_height_m=2, - target_height_m='invalid_type') - - # test error raising for mismatched dimensions with Series - with pytest.raises(ValueError): - bw.scale_air_pressure_to_height( - ref_air_pressure_hPa=pd.Series([1, 2, 3]), - ref_air_temp_degC=pd.Series([1, 2]), - ref_height_m=2, - target_height_m=10) - -def test_scale_air_density_to_height(): - assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) == 1.22174 - - assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, - ref_height_m=80, - target_height_m=100, - lapse_rate_kg_m3_m=-0.0002) == 1.22 - - test_density = bw.calc_air_density(DATA.T2m, DATA.P2m) - pd.testing.assert_series_equal(bw.scale_air_density_to_height( - ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10), - pd.Series(data = [1.186780, 1.187175, 1.187747, 1.185950, 1.186301 , 1.185686], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - -def test_scale_air_temperature_to_height(): - assert bw.scale_air_temperature_to_height(10.0065, 10, 11) == 10.0 - - assert bw.scale_air_temperature_to_height(ref_air_temperature=10, - ref_height_m=12, - target_height_m=10, - lapse_rate_deg_m=-0.001) == 10.002 - - pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( - ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, - target_height_m=20, - lapse_rate_deg_m=-0.001), - pd.Series(data=[0.936, 0.845, 0.713, 0.834, 0.753, 0.895], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( - ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, - target_height_m=20), - pd.Series(data=[0.837, 0.746, 0.614, 0.735, 0.654, 0.796], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) diff --git a/tests/test_scale.py b/tests/test_scale.py new file mode 100644 index 00000000..f63d66dc --- /dev/null +++ b/tests/test_scale.py @@ -0,0 +1,160 @@ +import pytest +import brightwind as bw +import pandas as pd +import numpy as np + +DATA = bw.load_csv(bw.demo_datasets.demo_data) +DATA = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) + + +def test_scale_air_pressure_to_height(): + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=200), 2) == 977.45 + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=15), 2) == 999.4 + pd.testing.assert_series_equal(bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=DATA['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=DATA['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data=[933.07, 933.07, 933.07, 932.07, 932.07, 932.07], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + # test error raising for invalid input types + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa='invalid_type', + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m=10) + + # test error raising for invalid slope type + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=1000, + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m='invalid_type') + + # test error raising for mismatched dimensions with Series + with pytest.raises(ValueError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=pd.Series([1, 2, 3]), + ref_air_temp_degC=pd.Series([1, 2]), + ref_height_m=2, + target_height_m=10) + + +def test_scale_air_density_to_height(): + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) == 1.22174 + + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, + ref_height_m=80, + target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) == 1.22 + + test_density = bw.calc_air_density(DATA.T2m, DATA.P2m) + pd.testing.assert_series_equal(bw.scale_air_density_to_height( + ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data=[1.186780, 1.187175, 1.187747, 1.185950, 1.186301, 1.185686], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + +def test_scale_air_temperature_to_height(): + assert bw.scale_air_temperature_to_height(10.0065, 10, 11) == 10.0 + + assert bw.scale_air_temperature_to_height(ref_air_temperature=10, + ref_height_m=12, + target_height_m=10, + lapse_rate_deg_m=-0.001) == 10.002 + + pd.testing.assert_series_equal( + bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20, + lapse_rate_deg_m=-0.001), + pd.Series(data=[0.936, 0.845, 0.713, 0.834, 0.753, 0.895], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + pd.testing.assert_series_equal( + bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20), + pd.Series(data=[0.837, 0.746, 0.614, 0.735, 0.654, 0.796], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + +def test_linear_transform(): + # test with float and int inputs + assert bw.linear_transform(x_target=10, x_ref=5, y_ref=10, slope=-0.5) == 7.5 + + +# test with array input for y_ref +assert (bw.linear_transform( + x_target=20, + x_ref=2, + y_ref=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) == np.array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796])).all() + +# test with pandas Series input for y_ref +pd.testing.assert_series_equal( + bw.linear_transform( + x_target=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065).round(3), + pd.Series(data=[20.007, 20.007, 20.008, 20.007, 20.008, 20.007], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + +# test error raising for invalid input types +with pytest.raises(TypeError): + bw.linear_transform( + x_target='invalid_type', + x_ref=2, + y_ref=20, + slope=-0.0065) + +# test error raising for invalid slope type +with pytest.raises(TypeError): + bw.linear_transform( + x_target=10, + x_ref=2, + y_ref=20, + slope='invalid_type') + +# test error raising for mismatched dimensions +with pytest.raises(ValueError): + bw.linear_transform( + x_target=np.array([1, 2, 3]), + x_ref=np.array([1, 2]), + y_ref=20, + slope=-0.5) + +# test error raising for mismatched dimensions with Series +with pytest.raises(ValueError): + bw.linear_transform( + x_target=pd.Series([1, 2, 3]), + x_ref=2, + y_ref=pd.Series([1, 2]), + slope=-0.5) + + +def test_apply_scale_factor(): + assert bw.apply_scale_factor(3, 0.5) == 1.5 + assert (bw.apply_scale_factor(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() + assert (bw.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) == [-100, -200, -300, -400]).all() + df = pd.DataFrame({'a': [0.5, 1.2], 'b': [3, 4], 'c': ['a', 'b']}) + result_df = pd.DataFrame({'a': [1.0, 2.4], 'b': [6, 8], 'c': ['a', 'b']}) + assert result_df.equals(bw.apply_scale_factor(df, 2)) diff --git a/tests/test_transform.py b/tests/test_transform.py index 90a37fec..3a1c6ebb 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -804,8 +804,8 @@ def test_scale_wind_speed(): assert bw.scale_wind_speed(3, 0.5) == 1.5 assert (bw.scale_wind_speed(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() - assert (bw.scale_wind_speed(DATA_CLND['Spd40mN'].tail(3), 0.5)==[4.015 , 3.4055, 2.9325]).all() - - df = pd.DataFrame({'Spd_100m':[0.5, 1.2], 'Spd_100m':[3, 4], 'c':['a', 'b']}) - result_df = pd.DataFrame({'Spd_100m':[1.0, 2.4], 'Spd_100m':[6, 8], 'c':['a', 'b']}) + assert (bw.scale_wind_speed(DATA_CLND['Spd40mN'].tail(3), 0.5) == [4.015, 3.4055, 2.9325]).all() + + df = pd.DataFrame({'Spd_100m': [0.5, 1.2], 'Spd_101m': [3, 4], 'c': ['a', 'b']}) + result_df = pd.DataFrame({'Spd_100m': [1.0, 2.4], 'Spd_101m': [6, 8], 'c': ['a', 'b']}) assert result_df.equals(bw.scale_wind_speed(df, 2)) diff --git a/tests/test_utils.py b/tests/test_utils.py index a188573f..e48096b0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -26,66 +26,3 @@ def test_slice_data(): data_sliced = bw.utils.utils.slice_data(DATA, date_to='2017-10-23') assert data_sliced.index[0] == DATA.index[0] - - -def test_linear_transform(): - # test with float and int inputs - assert bw.utils.utils.linear_transform(x_target=10, x_ref=5, y_ref=10, slope=-0.5) == 7.5 - - # test with array input for y_ref - assert (bw.utils.utils.linear_transform( - x_target=20, - x_ref=2, - y_ref=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, - slope=-0.0065) == np.array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796])).all() - - # test with pandas Series input for y_ref - pd.testing.assert_series_equal(bw.utils.utils.linear_transform( - x_target=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - x_ref=2, - y_ref=20, - slope=-0.0065).round(3), - pd.Series(data = [20.007, 20.007, 20.008, 20.007, 20.008, 20.007], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - # test error raising for invalid input types - with pytest.raises(TypeError): - bw.utils.utils.linear_transform( - x_target='invalid_type', - x_ref=2, - y_ref=20, - slope=-0.0065) - - #test error raising for invalid slope type - with pytest.raises(TypeError): - bw.utils.utils.linear_transform( - x_target=10, - x_ref=2, - y_ref=20, - slope='invalid_type') - - #test error raising for mismatched dimensions - with pytest.raises(ValueError): - bw.utils.utils.linear_transform( - x_target=np.array([1, 2, 3]), - x_ref=np.array([1, 2]), - y_ref=20, - slope=-0.5) - - #test error raising for mismatched dimensions with Series - with pytest.raises(ValueError): - bw.utils.utils.linear_transform( - x_target=pd.Series([1, 2, 3]), - x_ref=2, - y_ref=pd.Series([1, 2]), - slope=-0.5) - -def test_apply_scale_factor(): - assert bw.utils.utils.apply_scale_factor(3, 0.5) == 1.5 - assert (bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() - assert (bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) == [-100, -200, -300, -400]).all() - df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) - result_df = pd.DataFrame({'a':[1.0, 2.4], 'b':[6, 8], 'c':['a', 'b']}) - assert result_df.equals(bw.utils.utils.apply_scale_factor(df, 2)) From 88e3c21ee73016c48f675e86001b718b59350951 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Fri, 28 Nov 2025 10:35:39 +0000 Subject: [PATCH 17/35] iss #520 return deadband offset results table (#551) * iss #520 return deadband offset table * iss #520 adding to changelog * iss #520 update to include height * iss #520 minor formatting * iss #520 move location in changelog * iss #520 response to comments * iss #520 improved docstring example --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 1 + brightwind/transform/transform.py | 56 ++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55720797..7efae53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Additional labels for pre-release and build metadata are available as extensions 3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) 4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) 5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) +1. Added optional output to `apply_wind_vane_deadband_offset()`, which provides a table of deadband offset, logger offset and applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) ## [2.3.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 509f687c..c51ce8c1 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1063,7 +1063,7 @@ def offset_wind_direction(wdir, offset: float): return wdir.add(offset).apply(utils._range_0_to_360) -def apply_wind_vane_deadband_offset(data, measurements, inplace=False): +def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_results_table=False): """ Automatically apply deadband offsets of the wind vanes to the timeseries data. The deadband orientation information for each wind direction measurement and time period is contained in the measurements @@ -1082,18 +1082,23 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): This function accounts for this adjustment. - :param data: Timeseries data. - :type data: pd.DataFrame or pd.Series - :param measurements: Measurement information extracted from a WRA Data Model using bw.MeasurementStation - :type measurements: list or dict or _Measurements - :param inplace: If 'inplace' is True, the original direction data, contained in 'data', will be - modified and replaced with the adjusted direction data. If 'inplace' is False, the - original data will not be touched and instead a new DataFrame containing the adjusted - direction data is created. To store this adjusted direction data, please ensure it is - assigned to a new variable. - :type inplace: bool - :return: Data with adjusted wind direction by the deadband orientation. - :rtype: pd.DataFrame or pd.Series + :param data: Timeseries data. + :type data: pd.DataFrame or pd.Series + :param measurements: Measurement information extracted from a WRA Data Model using bw.MeasurementStation + :type measurements: list or dict or _Measurements + :param inplace: If 'inplace' is True, the original direction data, contained in 'data', will be + modified and replaced with the adjusted direction data. If 'inplace' is False, the + original data will not be touched and instead a new DataFrame containing the + adjusted direction data is created. To store this adjusted direction data, please + ensure it is assigned to a new variable. + :type inplace: bool + :param return_results_table: Optional key to return a dataframe containing deadband offset, logger offset and + applied offset for each directional sensor and the time period it is relevant for. + :type return_results_table: pd.DataFrame + :return: Data with adjusted wind direction by the deadband orientation, or where + return_results_table is specified, a tuple of the data and a DataFrame of + deadband offsets. + :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] **Example usage** :: @@ -1121,6 +1126,12 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): bw.apply_wind_vane_deadband_offset(data['Dir78mS'], mm1.measurements['Dir78mS'], inplace=True) print('\nWind vane deadband offset adjustment is completed.') + Specifying that the deadband offsets should be returned:: + data_adj, results_table = bw.apply_wind_vane_deadband_offset( + data['Dir78mS'], mm1.measurements['Dir78mS'], inplace=True, return_results_table=True + ) + results_table + """ # Depending on what is sent, get wdir properties into a list of properties wdirs_properties = _get_consistent_properties_format(measurements, 'wind_direction') @@ -1133,6 +1144,7 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): df = pd.DataFrame(data) if type(data) == pd.Series else data # Apply the offset + rows = [] for wdir_prop in wdirs_properties: name = wdir_prop['name'] if name in df.columns: @@ -1167,6 +1179,16 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): else: print('{} has dead_band_orientation of None from {} to {}.\n' .format(utils.bold(name), utils.bold(date_from), utils.bold(date_to_txt))) + height = wdir_prop.get('height_m') + rows.append({ + "name": name, + "height": height, + "deadband_offset": deadband, + "logger_offset": logger_offset, + "offset_applied": offset, + "date_from": date_from, + "date_to": date_to + }) else: print('{} is not found in data.\n'.format(utils.bold(name))) @@ -1175,6 +1197,14 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): # if a Series is sent, send back a Series if type(data) == pd.Series: df = df[df.columns[0]] + if return_results_table: + results_table = pd.DataFrame(rows).sort_values(by=["name", "date_from"]).groupby( + ["name", "height", "deadband_offset", "logger_offset", "offset_applied"] + ).agg({ + "date_from": "first", + "date_to": lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(level=[1, 2, 3, 4]) + return df, results_table return df From e8ae2ff18fd8b36f6f7443b639883041fd2aae27 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Fri, 28 Nov 2025 14:21:55 +0000 Subject: [PATCH 18/35] Iss520 rename table headers (#559) * iss #520 rename columns of output table * iss #520 removal of potentially problematic logic * iss #520 revised sorting logic * iss #520 improved logic for grouping neighbouring rows * iss #520 revised filtering and column name capitalisation --- brightwind/transform/transform.py | 45 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index c51ce8c1..eb4b2bd2 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1181,13 +1181,13 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re .format(utils.bold(name), utils.bold(date_from), utils.bold(date_to_txt))) height = wdir_prop.get('height_m') rows.append({ - "name": name, - "height": height, - "deadband_offset": deadband, - "logger_offset": logger_offset, - "offset_applied": offset, - "date_from": date_from, - "date_to": date_to + "Name": name, + "Height [m]": height, + "Vane Dead Band Orientation [deg]": deadband, + "Logger Offset": logger_offset, + "Offset Applied [deg]": offset, + "Date From": date_from, + "Date To": date_to }) else: print('{} is not found in data.\n'.format(utils.bold(name))) @@ -1198,12 +1198,31 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re if type(data) == pd.Series: df = df[df.columns[0]] if return_results_table: - results_table = pd.DataFrame(rows).sort_values(by=["name", "date_from"]).groupby( - ["name", "height", "deadband_offset", "logger_offset", "offset_applied"] - ).agg({ - "date_from": "first", - "date_to": lambda x: None if any(d is None for d in x) else max(x) - }).reset_index(level=[1, 2, 3, 4]) + results_df = pd.DataFrame(rows).sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + results_df['consecutive_group'] = ( + (results_df['Name'] != results_df['Name'].shift()) | + (results_df['Height [m]'] != results_df['Height [m]'].shift()) | + (results_df['Vane Dead Band Orientation [deg]'] != results_df['Vane Dead Band Orientation [deg]'].shift()) | + (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | + (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) + ).cumsum() + + # Group and aggregate consecutive periods + results_table = results_df.groupby([ + 'Name', + 'consecutive_group', + 'Height [m]', + 'Vane Dead Band Orientation [deg]', + 'Logger Offset', + 'Offset Applied [deg]' + ]).agg({ + 'Date From': 'first', + 'Date To': lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(drop=False).drop(columns=['consecutive_group']).set_index("Name").sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) return df, results_table return df From e9eb5e07bf71c7d1e446a5e7c09558ba6b705eb9 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Fri, 28 Nov 2025 14:32:13 +0000 Subject: [PATCH 19/35] Iss521 return device orientation offset table (#552) * iss #521 added optional output of device orientation table * iss #521 rename parameter for return_direction_offset_summary * iss #521 changes relating to comments on similar PRs * iss #521 groupby logic removed as not correct * iss #521 updated to take names from JSON schema * iss #521 change Name of measurement point * iss #521 revised sorting logic and added height as column * iss #521 update changelog * iss #521 revised grouping of rows logic * iss #521 formatting updates and docstring revision * iss #521 capitalisation of Applied * iss #521 fix PEP8 and expanded comment --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 4 +- brightwind/transform/transform.py | 79 ++++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7efae53b..1e3b0536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.4.0-dev] 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) + ### Bug Fixes 1. @@ -23,7 +24,8 @@ Additional labels for pre-release and build metadata are available as extensions 3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) 4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) 5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) -1. Added optional output to `apply_wind_vane_deadband_offset()`, which provides a table of deadband offset, logger offset and applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) +1. Added optional output to `apply_wind_vane_deadband_offset()` which provides a results table showing the applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) +1. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). ## [2.3.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index eb4b2bd2..b123aaa9 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1492,7 +1492,9 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals return df_copy.sort_index() -def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inplace=False): +def apply_device_orientation_offset( + data, measurement_station, wdir_cols=[], inplace=False, return_results_table=False + ): """ Applies a device orientation offset to wind direction data from remote sensing devices (lidar, sodar, or floating lidar) to align measurements with north. @@ -1522,18 +1524,21 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp Overlapping periods with non-zero `device_orientation_deg` values in `vertical_profiler_properties` are not supported and will raise an error. - :param data: Timeseries data. - :type data: pd.DataFrame or pd.Series - :param measurement_station: A simplified object to represent the IEA Wind Task 43 WRA Data Model. - :type measurement_station: bw.MeasurementStation - :param wdir_cols: Wind direction column names to apply the offset to. If empty, all wind direction - columns in the data are used. Default is an empty list. - :type wdir_cols: list - :param inplace: If True, modifies `data` in place. If False, returns a new DataFrame/Series - with adjusted values. Default is False. - :type inplace: bool, optional - :return: Data with wind direction adjusted by the orientation offset. - :rtype: pd.DataFrame or pd.Series + :param data: Timeseries data. + :type data: pd.DataFrame or pd.Series + :param measurement_station: A simplified object to represent the IEA Wind Task 43 WRA Data Model. + :type measurement_station: bw.MeasurementStation + :param wdir_cols: Wind direction column names to apply the offset to. If empty, all wind + direction columns in the data are used. Default is an empty list. + :type wdir_cols: list + :param inplace: If True, modifies `data` in place. If False, returns a new + DataFrame/Series with adjusted values. Default is False. + :type inplace: bool, optional + :param return_results_table: If True, returns a DataFrame containing the device orientation, + logger orientation and offset applied for each relevant time period. + :type return_results_table: bool, optional + :return: Data with wind direction adjusted by the orientation offset. + :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] **Example usage** :: @@ -1549,6 +1554,14 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp :: bw.apply_device_orientation_offset(data, fl1, inplace=True) print('Wind direction device orientation offset adjustment is completed.') + + Return the orientation offset results table along with adjusted timeseries data:: + :: + # Adjust only specific wind direction columns: + data_dev_orient_adj, results_table = bw.apply_device_orientation_offset( + data, fl1, wdir_cols=['Dir_40m', 'Dir_50m'], return_results_table=True + ) + results_table """ @@ -1580,6 +1593,7 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp _check_vertical_profiler_properties_overlap(measurement_station, df) + rows = [] # Apply the offset for i, wdir_prop in enumerate(wdirs_properties): name = wdir_prop['name'] @@ -1649,6 +1663,17 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp df[name] = _apply_dir_offset_target_orientation( df[name], logger_offset, device_orientation_deg, apply_offset_from, apply_offset_to, target_orientation_name='device orientation') + + height = wdir_prop.get('height_m') + rows.append({ + "Name": name, + "Height [m]": height, + "Device Orientation [deg]": device_orientation_deg, + "Logger Offset": logger_offset, + "Offset Applied [deg]": offset_wind_direction(device_orientation_deg, - logger_offset), + "Date From": apply_offset_from, + "Date To": apply_offset_to + }) else: wdir_not_in_dataset = True col_not_in_data.append(name) @@ -1669,6 +1694,34 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp if isinstance(data, pd.Series): df = df[df.columns[0]] data.update(df) + + if return_results_table: + results_df = pd.DataFrame(rows).sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + results_df['consecutive_group'] = ( + (results_df['Name'] != results_df['Name'].shift()) | + (results_df['Height [m]'] != results_df['Height [m]'].shift()) | + (results_df['Device Orientation [deg]'] != results_df['Device Orientation [deg]'].shift()) | + (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | + (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) + ).cumsum() + + # Group and aggregate consecutive periods with the same device orientation, logger offset and applied offset. + results_table = results_df.groupby([ + 'Name', + 'consecutive_group', + 'Height [m]', + 'Device Orientation [deg]', + 'Logger Offset', + 'Offset Applied [deg]' + ]).agg({ + 'Date From': 'first', + 'Date To': lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(drop=False).drop(columns=['consecutive_group']).set_index("Name").sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + return df, results_table return df From d5826fffbc198c6ed132d067e1a99a1c766498cf Mon Sep 17 00:00:00 2001 From: dancasey-ie Date: Fri, 28 Nov 2025 19:00:50 +0300 Subject: [PATCH 20/35] =?UTF-8?q?iss=20#550=20include=20method=20of=20auth?= =?UTF-8?q?enticating=20Brighthub=20requests=20using=20AP=E2=80=A6=20(#553?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * iss #550 include method of authenticating Brighthub requests using API keys. Included depreciation warnings for basic auth. Handle basic auth errors better. * Update docstrings * Update docstrings * iss #550 add entry to changelog * iss #550 PEP8, spelling and formatting updates * Change depreciation warnings to FutureWarnings * iss #550 Revert changelog changes, keep refresh token as depreciation warning * iss #550 update version number in warning * iss #550 put back in two missing changelog items --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 18 +-- brightwind/load/load.py | 263 +++++++++++++++++++++++++++------------- 2 files changed, 192 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e3b0536..e02535b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,17 @@ Additional labels for pre-release and build metadata are available as extensions 1. ### New Features and Enhancements -1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) -2. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) -3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) -4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) -5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) -1. Added optional output to `apply_wind_vane_deadband_offset()` which provides a results table showing the applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) -1. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). +1. Added `scale_air_pressure_to_height()` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height. ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) +2. Added `scale_air_density_to_height()` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height. ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) +3. Added `scale_air_temperature_to_height()` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height. ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) +4. Updated `calc_air_density()` to include relative humidity as suggested in IEC 61400-12-1. ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) + 1. Added deprecation warning the `specific_gas_constant` argument of `calc_air_density()` will be removed in v3.0. + 1. Added deprecation warning the scaling of air density to height within `calc_air_density()` will be removed in v3.0. Users should use `scale_air_density_to_height()` separately instead. +5. Added `apply_scale_factor()` to scale data by the scale_factor. ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) +6. Added optional output to `apply_wind_vane_deadband_offset()` which provides a results table showing the applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) +7. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). +8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) + 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. ## [2.3.0] diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 01f85d76..d3cffd3f 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -903,7 +903,6 @@ class _BrighthubAuth: This class is used to define general functions that are then called by LoadBrightHub. Functions in this class are outside of LoadBrightHub and will be called only once during the analysis and this will avoid making multiple login to the Brighthub user pool. - """ # List possible errors encountered on Login @@ -914,12 +913,61 @@ class are outside of LoadBrightHub and will be called only once during the analy "new_password_required": "Your password has expired or needs to be reset. " "Kindly reset your Brighthub password and try again.", "password_not_verified": "Could not verify your password. " - "Please ensure you have confirmed your email and the password is correct." + "Please ensure you have confirmed your email and the password is correct.", + "ms_sso_user": "Microsoft SSO users cannot authenticate using 'BRIGHTHUB_EMAIL' and 'BRIGHTHUB_PASSWORD'. " + "Please migrate to API key authentication." } ID_TOKEN = '' REFRESH_TOKEN = '' USERNAME = '' PASSWORD = '' + CLIENT_ID = '' + CLIENT_SECRET = '' + + class BrighthubAuthError(Exception): + """Custom exception for Brighthub authentication failures.""" + pass + + @staticmethod + def _authenticate_with_client_credentials(): + """ + Authenticate a Brighthub user using the OAuth2 client credentials flow. + + This method retrieves the Brighthub client ID and client secret from environmental + variables if not already assigned in the _BrighthubAuth class. + + :rtype: str + :return: The JWT ID token for authenticated requests. + """ + + if not _BrighthubAuth.CLIENT_ID: + _BrighthubAuth.CLIENT_ID = utils.get_environment_variable('BRIGHTHUB_CLIENT_ID') + + if not _BrighthubAuth.CLIENT_SECRET: + _BrighthubAuth.CLIENT_SECRET = utils.get_environment_variable('BRIGHTHUB_CLIENT_SECRET') + + base_uri = os.getenv('BRIGHTHUB_BASE_URI', 'https://api.brighthub.io') + auth_url = f"{base_uri}/auth/token" + try: + response = requests.post( + url=auth_url, + auth=requests.auth.HTTPBasicAuth(_BrighthubAuth.CLIENT_ID, _BrighthubAuth.CLIENT_SECRET), + data={"grant_type": "client_credentials"}, + timeout=10 + ) + response.raise_for_status() + except requests.exceptions.RequestException as e: + raise _BrighthubAuth.BrighthubAuthError(f"Network or connection error during authentication: {e}") + + try: + token_data = response.json() + except ValueError: + raise _BrighthubAuth.BrighthubAuthError(f"Invalid JSON response from Brighthub: {response.text}") + + if 'id_token' not in token_data: + raise _BrighthubAuth.BrighthubAuthError(f"Authentication failed, no id_token returned: {token_data}") + + return token_data['id_token'] @staticmethod def _get_cognito_request(): @@ -932,27 +980,33 @@ def _get_cognito_request(): 'Content-Type': 'application/x-amz-json-1.1' } client_id = os.getenv("BRIGHTHUB_USER_POOL_CLIENT_ID", "3qkkpikve578cbok46p136au3g") - # client_id = utils.get_environment_variable('BRIGHTHUB_USER_POOL_CLIENT_ID') return url, headers, client_id - + @staticmethod - def _get_id_token(): - """ - Function to login to the Brighthub user pool. - Assign a id_token and a refresh_token to the global variables ID_TOKEN, REFRESH_TOKEN which can be used to - make requests to the APIs. - In case of an error, a error message will be returned + def _authenticate_with_basic_auth(): + """Authenticate a Brighthub user using username and password (USER_PASSWORD_AUTH flow) + via Cognito and return the ID token and refresh token. + :rtype: tuple + :return: A tuple containing the JWT ID token and refresh token for authenticated requests. """ - url, headers, client_id = _BrighthubAuth._get_cognito_request() - + warnings.warn( + "Authentication using 'BRIGHTHUB_EMAIL' and 'BRIGHTHUB_PASSWORD' is deprecated in v2.4.0 and " + "will be removed in v3.0.0. \nPlease migrate to API key authentication. " + "Create and manage API keys at: https://brighthub.io/account-settings/settings. " + "After generating a key, set the environment variables 'BRIGHTHUB_CLIENT_ID' and " + "'BRIGHTHUB_CLIENT_SECRET'.", + FutureWarning + ) if not _BrighthubAuth.USERNAME: _BrighthubAuth.USERNAME = utils.get_environment_variable('BRIGHTHUB_EMAIL') if not _BrighthubAuth.PASSWORD: _BrighthubAuth.PASSWORD = utils.get_environment_variable('BRIGHTHUB_PASSWORD') + url, headers, client_id = _BrighthubAuth._get_cognito_request() + body = { "AuthParameters": { "USERNAME": _BrighthubAuth.USERNAME, @@ -961,34 +1015,86 @@ def _get_id_token(): "AuthFlow": "USER_PASSWORD_AUTH", "ClientId": client_id } - response = requests.post(url, headers=headers, json=body) login_response = response.json() + login_response_type = login_response.get("__type") # a login error occurred - if login_response.get("__type"): - if login_response["__type"] == "NotAuthorizedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) - elif login_response["__type"] == "UserNotConfirmedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) + if login_response_type: + if login_response_type == "NotAuthorizedException": + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) + elif login_response_type == "UserNotConfirmedException": + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) + elif 'This account is linked to Microsoft SSO' in login_response.get("message", ""): + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["ms_sso_user"]) else: - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) # challenge returned if login_response.get("ChallengeName"): if login_response["ChallengeName"] == "NEW_PASSWORD_REQUIRED": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["new_password_required"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["new_password_required"]) elif login_response["ChallengeName"] == "PASSWORD_VERIFIER": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["password_verifier"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["password_verifier"]) else: - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) # login successful id_token = login_response['AuthenticationResult']['IdToken'] refresh_token = login_response['AuthenticationResult']['RefreshToken'] + return id_token, refresh_token - _BrighthubAuth.ID_TOKEN = id_token - _BrighthubAuth.REFRESH_TOKEN = refresh_token + @staticmethod + def _get_id_token(): + """ + Retrieve an ID token for authenticating BrightHub API requests. + + This method attempts to authenticate using API client credentials if + `BRIGHTHUB_CLIENT_ID` and `BRIGHTHUB_CLIENT_SECRET` are available. If they + are not provided, it falls back to username/password authentication using + `BRIGHTHUB_EMAIL` and `BRIGHTHUB_PASSWORD`. + + Username/password authentication is deprecated in v2.4.0 and will be removed in + v3.0.0. Users should migrate to API key–based authentication. + + Upon success, this method stores the resulting ID token (and refresh token, + when applicable) in the class-level attributes `ID_TOKEN` and + `REFRESH_TOKEN`. In case of authentication failure, an appropriate + exception is raised by the underlying authentication method. + + :rtype: dict + :return: An empty dictionary upon successful authentication. + """ + if _BrighthubAuth.REFRESH_TOKEN: + try: + _BrighthubAuth._brighthub_refresh_token() + return {} + except Exception: + # If refresh fails, proceed to re-authenticate + pass + + # Preferred authentication flow: client credentials (API key) + try: + _BrighthubAuth.ID_TOKEN = _BrighthubAuth._authenticate_with_client_credentials() + return {} + + except Exception as e: + if str(e) in [ + f'BRIGHTHUB_CLIENT_ID environmental variable is not set.', + f'BRIGHTHUB_CLIENT_SECRET environmental variable is not set.' + ]: + # Fallback to basic auth if client credentials are not provided + # This method is depreciated in v2.4.0 and will be removed in v3.0.0 + _BrighthubAuth.ID_TOKEN, _BrighthubAuth.REFRESH_TOKEN = _BrighthubAuth._authenticate_with_basic_auth() + else: + raise e return {} @@ -997,17 +1103,22 @@ def _brighthub_refresh_token(): """ Function to generate a new token if the current id_token has expired. The new tokens are assigned to the global variables ID_TOKEN, REFRESH_TOKEN. - In case of an error, a error message will be returned - + In case of an error, an error message will be returned. """ - url, headers, client_id = _BrighthubAuth._get_cognito_request() + warnings.warn( + "Refresh token authentication is depreciated in v2.4.0 and will be removed in v3.0.0. " + "Please migrate to API key authentication.", + DeprecationWarning, + stacklevel=3 + ) + url, headers, cognito_client_id = _BrighthubAuth._get_cognito_request() body = { "AuthParameters": { "REFRESH_TOKEN": _BrighthubAuth.REFRESH_TOKEN }, "AuthFlow": "REFRESH_TOKEN_AUTH", - "ClientId": client_id + "ClientId": cognito_client_id } response = requests.post(url, headers=headers, json=body) login_response = response.json() @@ -1015,17 +1126,16 @@ def _brighthub_refresh_token(): # a login error occurred if login_response.get("__type"): if login_response["__type"] == "NotAuthorizedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) + raise ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) elif login_response["__type"] == "UserNotConfirmedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) + raise ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) else: - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) + raise ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) id_token = login_response['AuthenticationResult']['IdToken'] _BrighthubAuth.ID_TOKEN = id_token new_refresh_token = "" - # refresh token only expires after 30 days, it is not returned in the response if it is still valid if login_response['AuthenticationResult'].get('RefreshToken'): new_refresh_token = login_response['AuthenticationResult']['RefreshToken'] @@ -1039,20 +1149,20 @@ def _brighthub_refresh_token(): class LoadBrightHub: """ - LoadBrightHub allows you to pull meta data and timeseries data of measurements from the BrightHub + LoadBrightHub allows you to pull metadata and timeseries data of measurements from the BrightHub platform. This is a fast way to get access to the available open datasets on the platform. - To use LoadBrightHub, first sign up on www.brighthub.io and note your email and password. - - For security purposes LoadBrightHub uses stored environmental variables for your log in details. The - BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD environmental variables need to be set. In Windows this can be - done by opening the command prompt in Administrator mode and running: - - > setx BRIGHTHUB_EMAIL "your email") - > setx BRIGHTHUB_PASSWORD "your password") + To use LoadBrightHub: + 1. Create a BrightHub account at https://brighthub.io/auth/create-account. + 2. Create a new API key at https://brighthub.io/account-settings/settings. + and note your Client ID and Client Secret. + 3. Set the BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET as environmental variables. + In Windows this can be done by opening the command prompt in Administrator mode and running: + > setx BRIGHTHUB_CLIENT_ID "your API key client id") + > setx BRIGHTHUB_CLIENT_SECRET "your API key client secret") - If Anaconda or your Python environment is running you will need to restart it for the environmental variables to - take effect. + If Anaconda or your Python environment is running you will need to restart it for + the environmental variables to take effect. You can start by pulling all the available measurement stations available to you by running: @@ -1061,7 +1171,6 @@ class LoadBrightHub: """ __BASE_URI = os.getenv('BRIGHTHUB_BASE_URI', 'https://api.brighthub.io') - # __BASE_URI = utils.get_environment_variable('BRIGHTHUB_BASE_URI') @staticmethod def _brighthub_request(url_end, params=None): @@ -1075,11 +1184,9 @@ def _brighthub_request(url_end, params=None): :return response: The requests response object returned by requests.get() :rtype: requests.Response object """ - + if not _BrighthubAuth.ID_TOKEN: - login_response = _BrighthubAuth._get_id_token() - if "error" in login_response: - return ImportError(login_response) + _BrighthubAuth._get_id_token() url = "{}{}".format(LoadBrightHub.__BASE_URI, url_end) headers = {"authorization": _BrighthubAuth.ID_TOKEN} @@ -1088,19 +1195,11 @@ def _brighthub_request(url_end, params=None): # If there is an auth error due to expired token if response.status_code == 401 and response.json().get("message") == "The incoming token has expired": - # generate the token again - refresh_token_response = _BrighthubAuth._brighthub_refresh_token() - - # if an error occurred - if refresh_token_response.get("error"): - return ImportError(refresh_token_response) - else: - # token refreshed successfully - headers = {"authorization": _BrighthubAuth.ID_TOKEN} - - # make the request again - response = requests.get(url=url, headers=headers, params=params) - return response + # Authenticate again + _BrighthubAuth._get_id_token() + headers = {"authorization": _BrighthubAuth.ID_TOKEN} + # Make the request again + response = requests.get(url=url, headers=headers, params=params) return response @@ -1116,17 +1215,17 @@ def get_plants(plant_type=None, plant_uuid=None): :return: A table showing the available plants. :rtype: pd.DataFrame - To use LoadBrightHub, first sign up on www.brighthub.io and note your email and password. - - For security purposes LoadBrightHub uses stored environmental variables for your log in details. The - BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD environmental variables need to be set. In Windows this can be - done by opening the command prompt in Administrator mode and running: - - > setx BRIGHTHUB_EMAIL "your email") - > setx BRIGHTHUB_PASSWORD "your password") + To use LoadBrightHub: + 1. Create a BrightHub account at https://brighthub.io/auth/create-account. + 2. Create a new API key at https://brighthub.io/account-settings/settings. + and note your Client ID and Client Secret. + 3. Set the BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET as environmental variables. + In Windows this can be done by opening the command prompt in Administrator mode and running: + > setx BRIGHTHUB_CLIENT_ID "your API key client id") + > setx BRIGHTHUB_CLIENT_SECRET "your API key client secret") - If Anaconda or your Python environment is running you will need to restart it for the environmental variables to - take effect. + If Anaconda or your Python environment is running you will need to restart it for + the environmental variables to take effect. **Example usage** :: @@ -1192,17 +1291,17 @@ def get_measurement_stations(plant_uuid=None, measurement_station_uuid=None, mea :return: A table showing the available measurement stations. :rtype: pd.DataFrame | List[dict] - To use LoadBrightHub, first sign up on www.brighthub.io and note your email and password. - - For security purposes LoadBrightHub uses stored environmental variables for your log in details. The - BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD environmental variables need to be set. In Windows this can be - done by opening the command prompt in Administrator mode and running: - - > setx BRIGHTHUB_EMAIL "your email") - > setx BRIGHTHUB_PASSWORD "your password") - - If Anaconda or your Python environment is running you will need to restart it for the environmental variables to - take effect. + To use LoadBrightHub: + 1. Create a BrightHub account at https://brighthub.io/auth/create-account. + 2. Create a new API key at https://brighthub.io/account-settings/settings. + and note your Client ID and Client Secret. + 3. Set the BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET as environmental variables. + In Windows this can be done by opening the command prompt in Administrator mode and running: + > setx BRIGHTHUB_CLIENT_ID "your API key client id") + > setx BRIGHTHUB_CLIENT_SECRET "your API key client secret") + + If Anaconda or your Python environment is running you will need to restart it for + the environmental variables to take effect. **Example usage** :: From ba270302a1bfdad7eb9d1058e56546c4fdc0bfb4 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Mon, 1 Dec 2025 16:45:19 +0000 Subject: [PATCH 21/35] iss #545 added argument minimum_gap_length to time_continuity_gaps (#546) * iss #545 added argument minimum_gap_length to time_continuity_gaps * iss #545 update changelog * iss #545 added more filtering examples * iss #545 update spacing * iss #545 update doc string to clarify it is a 'time' gap * iss #545 update changelog --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 3 ++- brightwind/analyse/analyse.py | 34 +++++++++++++++++++++++++++------- tests/test_analyse.py | 6 ++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e02535b7..aec12fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0-dev] -1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)). ### Bug Fixes @@ -30,6 +30,7 @@ Additional labels for pre-release and build metadata are available as extensions 7. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). 8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. +1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 0c5d746c..3877f96b 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -1,5 +1,6 @@ import pandas as pd import numpy as np +from typing import Optional from brightwind.transform import transform as tf from brightwind.utils import utils from brightwind.analyse import plot as bw_plt @@ -1369,7 +1370,7 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 return graph -def time_continuity_gaps(data): +def time_continuity_gaps(data: pd.DataFrame, minimum_gap_length: Optional[pd.Timedelta] = None): """ Returns a table listing all the time gaps in the data that are not equal to the derived temporal resolution. @@ -1394,20 +1395,34 @@ def time_continuity_gaps(data): in the two available timestamps. It gives the actual amount of data missing e.g. if the two timestamps were 2020-01-01 01:10 and 2020-01-01 01:50 the days lost will equate to a 30 min of missing data and not 40 min. - :param data: Data for checking continuity, timestamp must be the index - :type data: pd.Series or pd.DataFrame - :return: A table listing all the time gaps in the data that are not equal to the derived - temporal resolution. - :rtype: pd.DataFrame + :param data: Data for checking continuity, timestamp must be the index + :type data: pd.Series or pd.DataFrame + :param minimum_gap_length: The minimum length time gap to report. Shorter time gaps will be filtered + out of the returned DataFrame + :type minimum_gap_length: Optional[pd.Timedelta] + :return: A table listing all the time gaps in the data that are not equal to the + derived temporal resolution. + :rtype: pd.DataFrame **Example usage** :: + import pandas as pd import brightwind as bw + data = bw.load_csv(bw.demo_datasets.demo_data) bw.time_continuity_gaps(data) bw.time_continuity_gaps(data['Spd80mN']) + # Removing all gaps shorter than 4hrs 30mins + bw.time_continuity_gaps(data['Spd80mN'], pd.Timedelta("4h 30min")) + + # Removing all gaps shorter than 20mins, expressed using alternative pandas notation + bw.time_continuity_gaps(data['Spd80mN'], pd.Timedelta(minutes=20)) + + # Removing all gaps shorter than 1day + bw.time_continuity_gaps(data['Spd80mN'], pd.Timedelta(days=1)) + """ indexes = data.dropna(how='all').index resolution = tf._get_data_resolution(indexes) @@ -1417,7 +1432,12 @@ def time_continuity_gaps(data): continuity = pd.DataFrame({'Date From': indexes.values.flatten()[:-1], 'Date To': indexes.values.flatten()[1:]}) - continuity['Days Lost'] = (continuity['Date To'] - continuity['Date From']) / pd.Timedelta('1 days') + continuity['Time gap'] = (continuity['Date To'] - continuity['Date From']) + + if minimum_gap_length: + continuity = continuity[continuity['Time gap'] >= minimum_gap_length] + + continuity['Days Lost'] = continuity['Time gap'] / pd.Timedelta('1 days') # Remove indexes where no days are lost before returning diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 2a3cf01a..bc1f5159 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -152,6 +152,12 @@ def test_time_continuity_gaps(): assert gaps.iloc[1, 1] == pd.Timestamp('2016-03-30 07:10:00') assert abs(gaps.iloc[0, 2] - 0.173611) < 1e-5 assert abs(gaps.iloc[1, 2] - 0.305556) < 1e-5 + + gaps = bw.time_continuity_gaps(DATA['Spd80mN'], pd.Timedelta("4h 30min")) + assert gaps.iloc[0, 0] == pd.Timestamp('2016-03-29 23:40:00') + assert gaps.iloc[0, 1] == pd.Timestamp('2016-03-30 07:10:00') + assert gaps.iloc[1, 0] == pd.Timestamp('2016-05-11 23:00:00') + assert gaps.iloc[1, 1] == pd.Timestamp('2016-05-31 15:20:00') # test for when timesteps are irregular # THIS WILL RAISE 3 WARNINGS. From 609374eddee05646eb5f96d08103fb13dff7ba3b Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:02:46 +0000 Subject: [PATCH 22/35] [Iss504] fix bug offset timestamps (#518) * iss #504 updated offset_timestamps function to include last timestamp when date_from unspecified * Revert "iss #504 updated offset_timestamps function to include last timestamp when date_from unspecified" This reverts commit f8040f691540dfc6d57ed6a30c6cbf3b8f62e6c2. * iss #504 updated offset_timestamps function to include last timestamp when date_to unspecified by setting date_to as last timestamp in dataframe + 10 mins * iss #504 added tests for this scenario to test_transform.py * iss #504 updated to add 1 second to the last timestamp instead of 10mins when no date_to given --------- Co-authored-by: Sara Rafter Co-authored-by: stephenholleran --- CHANGELOG.md | 4 ++-- brightwind/transform/transform.py | 4 ++-- tests/test_transform.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec12fed..5289a915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0-dev] -1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)). - +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) ### Bug Fixes 1. @@ -31,6 +30,7 @@ Additional labels for pre-release and build metadata are available as extensions 8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. 1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) +1. Updated `offset_timestamps()`to include last timestamp when date_to is unspecified, so that offset is applied to the entire record if date_to not specified. ([#504](https://github.com/brightwind-dev/brightwind/issues/504)) ## [2.3.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index b123aaa9..f9ce855d 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1441,9 +1441,9 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals if pd.isnull(date_to): if isinstance(data, pd.DatetimeIndex): - date_to = data[-1] + date_to = data[-1] + pd.DateOffset(seconds=1) elif isinstance(data, pd.Series) or isinstance(data, pd.DataFrame): - date_to = data.index[-1] + date_to = data.index[-1] + pd.DateOffset(seconds=1) else: date_to = pd.to_datetime(date_to) diff --git a/tests/test_transform.py b/tests/test_transform.py index 3a1c6ebb..932442d5 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -357,6 +357,24 @@ def test_offset_timestamps(): # sending index with no start end bw.offset_timestamps(series1.index, offset='90min') + series2 = DATA['2016-01-10 00:00:00':'2017-01-10 00:00:00'] + op = bw.offset_timestamps(series2.index, offset='90min') + assert len(op) == len(series2) + assert op[0] == pd.to_datetime('2016-01-10 01:30:00') + assert op[-1] == pd.to_datetime('2017-01-10 01:30:00') + + op = bw.offset_timestamps(series2.index, offset='-2H') + assert len(op) == len(series2) + assert op[0] == pd.to_datetime('2016-01-09 22:00:00') + assert op[-1] == pd.to_datetime('2017-01-09 22:00:00') + + # sending DataFrame with datetime index with no start end + op = bw.offset_timestamps(series2, offset='-10min') + assert (op.iloc[0] == series2.iloc[0]).all() + assert (op.iloc[-1] == series2.iloc[-1]).all() + assert len(op) == len(series2) + assert (op.loc['2017-01-09 23:50:00'] == series2.loc['2017-01-10 00:00:00']).all() + # sending index with start end op = bw.offset_timestamps(series1.index, offset='2min', date_from='2016-01-10 00:10:00') assert op[0] == pd.to_datetime('2016-01-10 00:00:00') From 586fa6f14709460d92ea59001913f97b302d7057 Mon Sep 17 00:00:00 2001 From: rm-mol <93541751+rm-mol@users.noreply.github.com> Date: Mon, 15 Dec 2025 19:50:08 +0100 Subject: [PATCH 23/35] iss255 tackle deprecation warnings (#517) * Update python, numpy, pandas * WIP * WIP * WIP * WIP * WIP * github actions only for Python 3.12 * add env variable * add tests.yml from dev branch * fix PEP8 missing whitespace after comma * missed PEP 8 whitespace * add backward compatibility and deprecating warnings for A, AS and H * add deprecating warning for T * add deprecating warning for S and backward compatibility * remove T from acceptable list * add msg of remaining actions * iss #255 compatibility testing and inclusion of dataframe_map * iss #255 add deprecation warning to `dataframe_map` function * iss #255 revised stacklevel of warning * iss #255 handling previously missed deprecation on time strings * iss #255 tidied comments * iss #255 updated docstring * iss #255 remove deprecated test * only convert string if necessary * changelog edit required * update changelog --------- Co-authored-by: stephenholleran Co-authored-by: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Co-authored-by: Olivia Bentley --- CHANGELOG.md | 2 + brightwind/analyse/analyse.py | 29 ++- brightwind/analyse/correlation.py | 39 +-- brightwind/analyse/plot.py | 4 +- brightwind/analyse/shear.py | 10 +- brightwind/load/load.py | 6 +- brightwind/load/station.py | 2 +- brightwind/transform/transform.py | 227 ++++++++++++++++-- .../how_to_get_some_useful_stats.ipynb | 4 +- .../how_to_transform_your_data.ipynb | 8 +- setup.py | 3 +- tests/test_analyse.py | 10 +- tests/test_correlation.py | 27 ++- tests/test_shear.py | 8 +- tests/test_transform.py | 128 +++++++--- 15 files changed, 378 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5289a915..30ed7d16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.4.0-dev] 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) + ### Bug Fixes 1. diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 3877f96b..bce9e967 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -154,10 +154,11 @@ def custom_agg(x): axis=1).dropna() if aggregation_method == '%frequency': - counts = data.groupby([y_series.name, x_series.name]).count().unstack(level=-1) + counts = data.groupby([y_series.name, x_series.name], observed=False).count().unstack(level=-1) distribution = counts / (counts.sum().sum()) * 100.0 else: - distribution = data.groupby([y_series.name, x_series.name]).agg(aggregation_method).unstack(level=-1) + distribution = data.groupby([y_series.name, x_series.name], + observed=False).agg(aggregation_method).unstack(level=-1) if y_bin_labels is not None: distribution.index = y_bin_labels @@ -390,7 +391,7 @@ def _mean_of_monthly_means_basic_method(df: pd.Series) -> pd.Series: Return a Series of mean of monthly mean with timestamp as the index. Calculate the monthly mean for each calendar month and then average the resulting 12 months. """ - mean_monthly_mean: pd.Series = df.groupby(df.index.month).mean().mean() + mean_monthly_mean: pd.Series = df.groupby(df.index.month, observed=False).mean().mean() return mean_monthly_mean @@ -618,9 +619,10 @@ def _derive_distribution(var_to_bin, var_to_bin_against, bins=None, aggregation_ data = pd.concat([var_to_bin.rename('data'), var_binned_series], join='inner', axis=1) if aggregation_method == '%frequency': - distribution = data.groupby(['variable_bin'])['data'].count().rename('%frequency') / len(data) * 100.0 + distribution = data.groupby(['variable_bin'], + observed=False)['data'].count().rename('%frequency') / len(data) * 100.0 else: - distribution = data.groupby(['variable_bin'])['data'].agg(aggregation_method) + distribution = data.groupby(['variable_bin'], observed=False)['data'].agg(aggregation_method) return distribution @@ -883,9 +885,9 @@ def dist_by_dir_sector(var_series, direction_series, sectors=12, aggregation_met _get_direction_binned_series(sectors, direction_series, direction_bin_array, direction_bin_labels) data = pd.concat([var_series.rename('data'), direction_binned_series], join='inner', axis=1) if aggregation_method == '%frequency': - result = data.groupby(['direction_bin'])['data'].count().rename('%frequency')/len(data) * 100.0 + result = data.groupby(['direction_bin'], observed=False)['data'].count().rename('%frequency')/len(data) * 100.0 else: - result = data.groupby(['direction_bin'])['data'].agg(aggregation_method) + result = data.groupby(['direction_bin'], observed=False)['data'].agg(aggregation_method) for i in range(1, sectors+1): if not (i in result.index): @@ -923,10 +925,11 @@ def _get_dist_matrix_by_dir_sector(var_series, var_to_bin_series, direction_seri data = pd.concat([var_series.rename('var_data'), var_binned_series, direction_binned_series], axis=1).dropna() if aggregation_method == '%frequency': - counts = data.groupby([var_to_bin_series.name, 'direction_bin']).count().unstack(level=-1) + counts = data.groupby([var_to_bin_series.name, 'direction_bin'], observed=False).count().unstack(level=-1) distribution = counts/(counts.sum().sum()) * 100.0 else: - distribution = data.groupby([var_to_bin_series.name, 'direction_bin']).agg(aggregation_method).unstack(level=-1) + distribution = data.groupby([var_to_bin_series.name, 'direction_bin'], + observed=False).agg(aggregation_method).unstack(level=-1) distribution.columns = distribution.columns.droplevel(0) for i in range(1, sectors + 1): if not (i in distribution.columns): @@ -1482,11 +1485,11 @@ def coverage(data, period='1M', aggregation_method='mean', data_resolution=None) :param period: Groups data by the period specified here. The following formats are supported - Set period to 10min for 10 minute average, 20min for 20 minute average and so on for 4min, 15min, etc. - - Set period to 1H for hourly average, 3H for three hourly average and so on for 5H, 6H etc. + - Set period to 1h for hourly average, 3h for three hourly average and so on for 5h, 6h etc. - Set period to 1D for a daily average, 3D for three day average, similarly 5D, 7D, 15D etc. - Set period to 1W for a weekly average, 3W for three week average, similarly 2W, 4W etc. - Set period to 1M for monthly average - - Set period to 1AS fo annual average + - Set period to 1YS fo annual average - Can be a DateOffset object too :type period: str or pandas.DateOffset @@ -1506,10 +1509,10 @@ def coverage(data, period='1M', aggregation_method='mean', data_resolution=None) data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_site_data) #To find hourly coverage - data_hourly = bw.coverage(data.Spd80mN, period='1H') + data_hourly = bw.coverage(data.Spd80mN, period='1h') #To find hourly coverage for multiple columns - data_hourly_multiple = bw.coverage(data[['Spd80mS','Spd60mN']], period='1H') + data_hourly_multiple = bw.coverage(data[['Spd80mS','Spd60mN']], period='1h') #To find monthly_coverage data_monthly = bw.coverage(data.Spd80mN, period='1M') diff --git a/brightwind/analyse/correlation.py b/brightwind/analyse/correlation.py index 9fe3e17e..63a0181e 100644 --- a/brightwind/analyse/correlation.py +++ b/brightwind/analyse/correlation.py @@ -220,9 +220,10 @@ def _rename_duplicated_columns(self): return self.ref_spd, self._ref_spd_col_name, self._ref_spd_col_names, self.ref_dir, self._ref_dir_col_name def _get_synth_start_dates(self): - none_even_freq = ['5H', '7H', '9H', '10H', '11H', '13H', '14H', '15H', '16H', '17H', '18H', '19H', - '20H', '21H', '22H', '23H', 'D', 'W'] + none_even_freq = ['5h', '7h', '9h', '10h', '11h', '13h', '14h', '15h', '16h', '17h', '18h', '19h', + '20h', '21h', '22h', '23h', 'D', 'W'] if any(freq in self.averaging_prd for freq in none_even_freq): + self.averaging_prd = tf._normalize_freq_string(self.averaging_prd) ref_time_array = pd.date_range(start=self.data.index[0], freq='-' + self.averaging_prd, end=self.ref_spd.index[0]) if ref_time_array.empty: @@ -323,11 +324,11 @@ class OrdinaryLeastSquares(CorrelBase): :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -405,7 +406,7 @@ class OrdinaryLeastSquares(CorrelBase): # Correlate temperature on an hourly basis using a different aggregation method. ols_cor = bw.Correl.OrdinaryLeastSquares(m2_ne['T2M_degC'], data['T2m'], - averaging_prd='1H', coverage_threshold=0, + averaging_prd='1h', coverage_threshold=0, ref_aggregation_method='min', target_aggregation_method='min') # Correlate wind speeds on a monthly basis and force the intercept through the origin. @@ -419,7 +420,7 @@ class OrdinaryLeastSquares(CorrelBase): # Correlate by directional sector forcing the intercept through the origin. ols_cor = bw.Correl.OrdinaryLeastSquares(m2_ne['WS50m_m/s'], data['Spd80mN'], - ref_dir=m2_ne['WD50m_deg'], averaging_prd='1H', + ref_dir=m2_ne['WD50m_deg'], averaging_prd='1h', coverage_threshold=0.9, forced_intercept_origin=True) """ def __init__(self, ref_spd, target_spd, averaging_prd, coverage_threshold=0.9, ref_dir=None, sectors=12, @@ -462,7 +463,7 @@ def run(self, show_params=True): elif type(self.ref_dir) is pd.Series: self.params = [] for sector, group in pd.concat([self.data, self._ref_dir_bins], - axis=1, join='inner').dropna().groupby('ref_dir_bin'): + axis=1, join='inner').dropna().groupby('ref_dir_bin', observed=False): # print('Processing sector:', sector) if len(group) > 1: slope, offset = self._leastsquare(ref_spd=group[self._ref_spd_col_name], @@ -514,11 +515,11 @@ class OrthogonalLeastSquares(CorrelBase): :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -583,7 +584,7 @@ class OrthogonalLeastSquares(CorrelBase): # Correlate temperature on an hourly basis using a different aggregation method. orthog_cor = bw.Correl.OrthogonalLeastSquares(m2_ne['T2M_degC'], data['T2m'], - averaging_prd='1H', coverage_threshold=0, + averaging_prd='1h', coverage_threshold=0, ref_aggregation_method='min', target_aggregation_method='min') """ @@ -638,11 +639,11 @@ class MultipleLinearRegression(CorrelBase): :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -705,7 +706,7 @@ class MultipleLinearRegression(CorrelBase): # Correlate temperature on an hourly basis using a different aggregation method. mul_cor = bw.Correl.MultipleLinearRegression([m2_ne['T2M_degC'], m2_nw['T2M_degC']], data['T2m'], - averaging_prd='1H', coverage_threshold=0, + averaging_prd='1h', coverage_threshold=0, ref_aggregation_method='min', target_aggregation_method='min') """ @@ -905,11 +906,11 @@ def __init__(self, ref_spd, ref_dir, target_spd, target_dir, averaging_prd, cove :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -936,7 +937,7 @@ def __init__(self, ref_spd, ref_dir, target_spd, target_dir, averaging_prd, cove # Basic usage on an hourly basis ss_cor = bw.Correl.SpeedSort(m2['WS50m_m/s'], m2['WD50m_deg'], data['Spd80mN'], data['Dir78mS'], - averaging_prd='1H') + averaging_prd='1h') ss_cor.run() ss_cor.plot_wind_directions() ss_cor.get_result_table() @@ -944,7 +945,7 @@ def __init__(self, ref_spd, ref_dir, target_spd, target_dir, averaging_prd, cove # Sending an array of direction sectors ss_cor = bw.Correl.SpeedSort(m2['WS50m_m/s'], m2['WD50m_deg'], data['Spd80mN'], data['Dir78mS'], - averaging_prd='1H', direction_bin_array=[0,90,130,200,360]) + averaging_prd='1h', direction_bin_array=[0,90,130,200,360]) ss_cor.run() """ @@ -1018,7 +1019,7 @@ def run(self, show_params=True): self.params['target_veer_cutoff'] = round(self.target_veer_cutoff, 5) self.params['overall_average_veer'] = round(self.overall_veer, 5) for sector, group in pd.concat([self.data, self._ref_dir_bins], - axis=1, join='inner').dropna().groupby('ref_dir_bin'): + axis=1, join='inner').dropna().groupby('ref_dir_bin', observed=False): # print('Processing sector:', sector) self.speed_model[sector] = SpeedSort.SectorSpeedModel(ref_spd=group[self._ref_spd_col_name], target_spd=group[self._tar_spd_col_name], @@ -1210,7 +1211,7 @@ def _predict(self, x_spd, x_dir): direction_bin_array=self.direction_bin_array).rename('ref_dir_bin')], axis=1, join='inner').dropna() prediction = pd.Series(dtype='float64').rename('spd') - for sector, data in x.groupby('ref_dir_bin'): + for sector, data in x.groupby('ref_dir_bin', observed=False): if sector in list(self.speed_model.keys()): prediction_spd = self.speed_model[sector].sector_predict(data['spd']) else: diff --git a/brightwind/analyse/plot.py b/brightwind/analyse/plot.py index 95303b59..2063f9f9 100644 --- a/brightwind/analyse/plot.py +++ b/brightwind/analyse/plot.py @@ -1971,7 +1971,7 @@ def plot_shear_by_sector(scale_variable, wind_rose_data, calc_method='power_law' if calc_method == 'log_law': label = 'Mean_Roughness_Coefficient' - scale_variable_y = np.append(scale_variable, scale_variable[0]) + scale_variable_y = np.append(scale_variable, scale_variable.iloc[0]) plot_x = np.append(radians, radians[0]) scale_to_fit = max(scale_variable[np.isfinite(scale_variable)]) / max(result / 100) wind_rose_r = (result / 100) * scale_to_fit @@ -2352,7 +2352,7 @@ def plot_shear_time_of_day(df, calc_method, plot_type='step'): ax.set_ylabel(label) # create x values for plot - idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1H').hour + idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1h').hour if plot_type == 'step': df = df.shift(+1, axis=0) diff --git a/brightwind/analyse/shear.py b/brightwind/analyse/shear.py index 592147f9..f49f5101 100644 --- a/brightwind/analyse/shear.py +++ b/brightwind/analyse/shear.py @@ -1044,13 +1044,13 @@ def _apply(self, wspds, height, shear_to, wdir=None): if self.calc_method == 'power_law': scaled_wspds[i] = Shear._scale(wspds=by_sector[i]['Unscaled_Wind_Speeds'], height=height, shear_to=shear_to, - calc_method=self.calc_method, alpha=self.alpha[i]) + calc_method=self.calc_method, alpha=self.alpha.iloc[i]) elif self.calc_method == 'log_law': scaled_wspds[i] = Shear._scale(wspds=by_sector[i]['Unscaled_Wind_Speeds'], height=height, shear_to=shear_to, calc_method=self.calc_method, - roughness=self._roughness[i]) + roughness=self._roughness.iloc[i]) if i == 0: result = scaled_wspds[i] @@ -1093,7 +1093,7 @@ def _fill_df_12x24(data): df_copy = data.copy() interval = int(24 / len(data)) # set index for new data frame to deal with less than 24 sectors - idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1H') + idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1h') # create new dataframe with 24 rows only interval number of unique values df = pd.DataFrame({cols: [np.nan] for cols in df_copy.columns}, index=pd.DatetimeIndex(idx).time) @@ -1130,13 +1130,13 @@ def _data_prep(wspds, heights, min_speed, maximise_data=False, return_raw_wspds= Shear._valid_wsp_data_error_msg(wspds, min_speed) if isinstance(wspds.index, pd.DatetimeIndex): if maximise_data is False: - cvg = coverage(wspds[wspds > min_speed].dropna(), period='1AS').sum()[1] + cvg = coverage(wspds[wspds > min_speed].dropna(), period='1YS').sum().iloc[1] else: _wspds = wspds[wspds > min_speed] count = _wspds.count(axis=1) count = count[count >= 2] count.rename('count', inplace=True) - cvg = coverage(count, period='1AS').sum() + cvg = coverage(count, period='1YS').sum() if not return_raw_wspds: wspds = wspds[wspds > min_speed].dropna() return wspds, cvg diff --git a/brightwind/load/load.py b/brightwind/load/load.py index d3cffd3f..7f02f700 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -1362,7 +1362,7 @@ def get_measurement_stations(plant_uuid=None, measurement_station_uuid=None, mea ) if return_df: - meas_loc_df = pd.read_json(json.dumps(meas_loc_json)) + meas_loc_df = pd.read_json(StringIO(json.dumps(meas_loc_json))) required_cols = ['name', 'measurement_station_type_id', 'latitude_ddeg', 'longitude_ddeg', 'plant_uuid', 'uuid', 'notes'] meas_loc_df = meas_loc_df[required_cols] @@ -2422,7 +2422,7 @@ def apply_cleaning(data, cleaning_file_or_df, inplace=False, sensor_col_name='Se else: for col in data.columns: if col.find(cleaning_df[sensor_col_name][k]) == 0: - data[col][(data.index >= date_from) & (data.index < date_to)] = replacement_text + data.loc[(data.index >= date_from) & (data.index < date_to), col] = replacement_text pd.options.mode.chained_assignment = 'warn' return data @@ -2651,7 +2651,7 @@ def apply_cleaning_windographer(data, windog_cleaning_file, inplace=False, flags for col in data.columns: if col.find(cleaning_df[sensor_col_name][k]) == 0: if cleaning_df[flag_col_name][k] not in flags_to_exclude: - data[col][(data.index >= date_from) & (data.index < date_to)] = replacement_text + data.loc[(data.index >= date_from) & (data.index < date_to), col] = replacement_text pd.options.mode.chained_assignment = 'warn' return data diff --git a/brightwind/load/station.py b/brightwind/load/station.py index 3a9bd29c..889ee7ab 100644 --- a/brightwind/load/station.py +++ b/brightwind/load/station.py @@ -903,7 +903,7 @@ def __get_table_for_cols(self, columns_to_show): temp_df.sort_values(['name', 'date_from'], ascending=[True, True], inplace=True) temp_df.fillna('-', inplace=True) # groupby drops nan so need to fill them in # group duplicate data for the columns available - grouped_by_avail_cols = temp_df.groupby(avail_cols) + grouped_by_avail_cols = temp_df.groupby(avail_cols, observed=False) # get date_to from the last row in each group to assign to the first row. new_date_to = grouped_by_avail_cols.last()['date_to'] df = grouped_by_avail_cols.first()[['date_from', 'date_to']] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index f9ce855d..d8b53e49 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -24,6 +24,12 @@ 'apply_wspd_slope_offset_adj', 'apply_device_orientation_offset'] +_warned_a = False # warning for pandas 'A' frequency string so only shows once +_warned_as = False # warning for pandas 'AS' frequency string so only shows once +_warned_h = False # warning for pandas 'H' frequency string so only shows once +_warned_t = False # warning for pandas 'T' frequency string so only shows once +_warned_s = False # warning for pandas 'S' frequency string so only shows once + def _compute_wind_vector(wspd, wdir): """ @@ -32,6 +38,120 @@ def _compute_wind_vector(wspd, wdir): return wspd*np.cos(wdir), wspd*np.sin(wdir) +def dataframe_map(df, func, **kwargs): + """ + Apply a function element-wise to a DataFrame. + + Compatibility wrapper for DataFrame.map() (pandas >=2.1) and + DataFrame.applymap() (pandas <2.1). + + :param df: DataFrame to apply function to + :type df: pd.DataFrame + :param func: Function to apply element-wise + :type func: Callable + :return: Transformed DataFrame + :rtype: pd.DataFrame + """ + if hasattr(df, 'map'): + # pandas >= 2.1 + return df.map(func, **kwargs) + else: + # pandas < 2.1 + warnings.warn( + "Support for pandas versions <2.1 is ending in a future brightwind version.", + DeprecationWarning, + stacklevel=2 + ) + return df.applymap(func, **kwargs) + + +def _normalize_freq_string(period): + """ + Convert a deprecated pandas frequency string to its modern equivalent. + + Pandas frequency strings are available here: + https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects + + :param period: Frequency string that may or may not be deprecated + :type period: str + :return: Frequency string in it's modern equivalent form where necessary. + :rtype: str + """ + global _warned_a, _warned_as, _warned_h, _warned_t, _warned_s + pandas_version = tuple(map(int, pd.__version__.split('.')[:2])) + + # Handle deprecated 'A' -> 'Y' (but 'YS' is preferred for year-start) + if period.endswith('A') and not period.endswith('BA'): + if not _warned_a: + warnings.warn( + "'A' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version.", + DeprecationWarning, + stacklevel=3 + ) + _warned_a = True + if pandas_version < (2, 2): + return period[:-1] + 'YS' + + # Handle deprecated 'AS' -> 'YS' + if period.endswith('AS'): + if not _warned_as: + warnings.warn( + "'AS' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'YS' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_as = True + if pandas_version < (2, 2): + return period[:-2] + 'YS' + + # Handle deprecated 'H' -> 'h' + if period.endswith('H'): + if not _warned_h: + warnings.warn( + "'H' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'h' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_h = True + if pandas_version < (2, 2): + return period[:-1] + 'h' + + # Handle deprecated 'T' -> 'min' + if period.endswith('T'): + if not _warned_t: + warnings.warn( + "'T' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'min' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_t = True + if pandas_version < (2, 2): + return period[:-1] + 'min' + + # Handle deprecated 'S' -> 's' (but not 'MS', 'YS', 'AS', etc.) + if period.endswith('S') and not any(period.endswith(x) for x in ['MS', 'YS', 'AS', 'NS']): + if not _warned_s: + warnings.warn( + "'S' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 's' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_s = True + if pandas_version < (2, 2): + return period[:-1] + 's' + + return period + + def _freq_str_to_dateoffset(period): """ Convert a pandas frequency string to a pd.DateOffset. @@ -44,38 +164,96 @@ def _freq_str_to_dateoffset(period): :return: A pd.DateOffset :rtype: pd.DateOffset """ + global _warned_a + global _warned_as + global _warned_h + global _warned_t + global _warned_s + if period[-1] == 'M': as_dateoffset = pd.DateOffset(months=int(period[:-1])) elif period[-2:] == 'MS': as_dateoffset = pd.DateOffset(months=int(period[:-2])) elif period[-1] == 'A': + if not _warned_a: + warnings.warn( + "'A' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version.", + DeprecationWarning, + stacklevel=3 + ) + _warned_a = True as_dateoffset = pd.DateOffset(years=float(period[:-1])) elif period[-2:] == 'AS': + if not _warned_as: + warnings.warn( + "'AS' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'YS' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_as = True + as_dateoffset = pd.DateOffset(years=float(period[:-2])) + elif period[-2:] == 'YS': as_dateoffset = pd.DateOffset(years=float(period[:-2])) elif period[-1:] == 'W': as_dateoffset = pd.DateOffset(weeks=float(period[:-1])) elif period[-1:] == 'D': as_dateoffset = pd.DateOffset(days=float(period[:-1])) elif period[-1:] == 'H': + if not _warned_h: + warnings.warn( + "'H' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'h' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_h = True + as_dateoffset = pd.DateOffset(hours=float(period[:-1])) + elif period[-1:] == 'h': as_dateoffset = pd.DateOffset(hours=float(period[:-1])) elif period[-1:] == 'T': + if not _warned_t: + warnings.warn( + "'T' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'min' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_t = True as_dateoffset = pd.DateOffset(minutes=float(period[:-1])) elif period[-3:] == 'min': as_dateoffset = pd.DateOffset(minutes=float(period[:-3])) elif period[-1:] == 'S': + if not _warned_s: + warnings.warn( + "'S' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 's' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_s = True + as_dateoffset = pd.DateOffset(seconds=float(period[:-1])) + elif period[-1:] == 's': as_dateoffset = pd.DateOffset(seconds=float(period[:-1])) else: - raise ValueError('"{}" period not recognized. Only units "M", "MS", "A", "AS", "W", "D", "H", "T", "min", "S" ' + raise ValueError('"{}" period not recognized. Only units "M", "MS", "YS", "W", "D", "h", "min", "s" ' 'are recognized'.format(period)) + + return as_dateoffset def _convert_days_to_hours(prd): - return str(int(prd[:-1])*24)+'H' + return str(int(prd[:-1])*24)+'h' def _convert_weeks_to_hours(prd): - return str(int(prd[:-1])*24*7)+'H' + return str(int(prd[:-1])*24*7)+'h' def _get_min_overlap_timestamp(df1_timestamps, df2_timestamps): @@ -174,7 +352,7 @@ def _round_timestamp_down_to_averaging_prd(timestamp, period): :param timestamp: Timestamp to round down from. :type timestamp: pd.Timestamp - :param period: Averaging period e.g. '10min', '1H', '3H', '6H', '1D', '7D', '1W', '1MS', '1AS' + :param period: Averaging period e.g. '10min', '1h', '3h', '6h', '1D', '7D', '1W', '1MS', '1YS' :type period: str :return: Timestamp to represent the start of an averaging period which covers the timestamp. :rtype: str @@ -188,12 +366,13 @@ def _round_timestamp_down_to_averaging_prd(timestamp, period): if 1M, 1MS it should go to start of month if 1A, 1AS it should go to start of year """ + period = _normalize_freq_string(period) if period[-3:] == 'min': return '{year}-{month}-{day} {hour}:{minute}:00'.format(year=timestamp.year, month=timestamp.month, day=timestamp.day, hour=timestamp.hour, minute=_round_down_to_multiple(timestamp.minute, int(period[:-3]))) - elif period[-1] == 'H': + elif period[-1] == 'h' or period[-1] == 'H': return '{year}-{month}-{day} {hour}:00:00'.format(year=timestamp.year, month=timestamp.month, day=timestamp.day, hour=_round_down_to_multiple(timestamp.hour, int(period[:-1]))) @@ -202,7 +381,7 @@ def _round_timestamp_down_to_averaging_prd(timestamp, period): hour=timestamp.hour) elif period[-1] == 'M' or period[-2:] == 'MS': return '{year}-{month}'.format(year=timestamp.year, month=timestamp.month) - elif period[-2:] == 'AS' or period[-1:] == 'A': + elif period[-2:] == 'YS' or period[-1:] == 'A' or period[-2:] == 'AS': return '{year}'.format(year=timestamp.year) else: print("Warning: Averaging period not identified returning default timestamps") @@ -337,11 +516,11 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met :param period: Groups data by the period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type period: str :param wdir_column_names: List of wind direction column names. These columns, if the aggregation_method is mean, @@ -376,7 +555,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met data = bw.load_csv(bw.demo_datasets.demo_data) # To find hourly averages - data_hourly = bw.average_data_by_period(data.Spd80mN, period='1H') + data_hourly = bw.average_data_by_period(data.Spd80mN, period='1h') # To find monthly averages data_monthly = bw.average_data_by_period(data.Spd80mN, period='1M') @@ -397,6 +576,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met """ coverage_threshold = validate_coverage_threshold(coverage_threshold) + period = _normalize_freq_string(period) if isinstance(period, str): if period[-1] == 'D': @@ -408,7 +588,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met if period[-1] == 'A': period = period+'S' if period[-1] == 'Y': - raise TypeError("Please use '1AS' for annual frequency at the start of the year.") + raise TypeError("Please use '1YS' for annual frequency at the start of the year.") # Check that the data resolution is not less than the period specified if data_resolution is None: @@ -416,8 +596,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met raise ValueError("The time period specified is less than the temporal resolution of the data. " "For example, hourly data should not be averaged to 10 minute data.") data = data.sort_index() - grouper_obj = data.resample(period, axis=0, closed='left', label='left', - convention='start', kind='timestamp') + grouper_obj = data.resample(period, closed='left', label='left') # if period is equal to data resolution then no need to vector average wind direction is_period_not_equal_to_resolution = (_freq_str_to_dateoffset(period) != _get_data_resolution(data.index)) @@ -542,7 +721,7 @@ def _vector_avg_of_wdirs_dataframe(wdirs, wspds=None): # means there is no wind direction => return NaN nan_mask = (avg_dir_df['sine'] == 0) & (avg_dir_df['cosine'] == 0) avg_dir_df['avg_dir'] = np.rad2deg(np.arctan2(sine, cosine)) % 360 - avg_dir_df['avg_dir'][nan_mask] = np.nan + avg_dir_df.loc[nan_mask,'avg_dir'] = np.nan return avg_dir_df['avg_dir'] @@ -705,11 +884,11 @@ def merge_datasets_by_period(data_1, data_2, period, :param period: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type period: str :param wdir_column_names_1: List of wind direction column names. These columns, if the aggregation_method is mean, @@ -1058,7 +1237,7 @@ def offset_wind_direction(wdir, offset: float): if isinstance(wdir, float) or isinstance(wdir, int): return utils._range_0_to_360(wdir + offset) elif isinstance(wdir, pd.DataFrame): - return wdir.add(offset).applymap(utils._range_0_to_360) + return dataframe_map(wdir.add(offset), utils._range_0_to_360) elif isinstance(wdir, pd.Series): return wdir.add(offset).apply(utils._range_0_to_360) @@ -1367,14 +1546,14 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals - Set offset to 10min to add 10 minutes to each timestamp, -10min to subtract 10 minutes and so on for 4min, 20min, etc. - - Set offset to 1H to add 1 hour to each timestamp and -1H to subtract and so on for 5H, 6H, + - Set offset to 1h to add 1 hour to each timestamp and -1h to subtract and so on for 5h, 6h, etc. - Set offset to 1D to add a day and -1D to subtract and so on for 5D, 7D, 15D, etc. - Set offset to 1W to add a week and -1W to subtract from each timestamp and so on for 2W, 4W, etc. - Set offset to 1M to add a month and -1M to subtract a month from each timestamp and so on for 2M, 3M, etc. - - Set offset to 1Y to add an year and -1Y to subtract an year from each timestamp and so on + - Set offset to 1Y to add an year and -1Y to subtract a year from each timestamp and so on for 2Y, 3Y, etc. :type offset: str @@ -1402,7 +1581,7 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals data = bw.load_csv(bw.demo_datasets.demo_data) # To decrease 10 minutes within a given date range and overwrite the original data - op1 = bw.offset_timestamps(data, offset='1H', date_from='2016-02-01 00:20:00', + op1 = bw.offset_timestamps(data, offset='1h', date_from='2016-02-01 00:20:00', date_to='2016-02-01 01:40:00', overwrite=True) # To decrease 10 minutes within a given date range not overwriting the original data @@ -1419,14 +1598,14 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals op4 = bw.offset_timestamps(data.index, offset='-10min', date_from='2016-02-01 00:20:00', date_to='2016-02-01 01:40:00') - # Can also except decimal values for offset, like 3.5H for 3 hours and 30 minutes - op5 = bw.offset_timestamps(data.index, offset='3.5H', date_from='2016-02-01 00:20:00', + # Can also except decimal values for offset, like 3.5h for 3 hours and 30 minutes + op5 = bw.offset_timestamps(data.index, offset='3.5h', date_from='2016-02-01 00:20:00', date_to='2016-02-01 01:40:00') # Can accept also Timestamp and datetime objects - bw.offset_timestamps(data.index[0], offset='4H') - bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5H') - bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5H') + bw.offset_timestamps(data.index[0], offset='4h') + bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5h') + bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5h') bw.offset_timestamps(datetime.time(0, 20), offset='30min') """ diff --git a/docs/source/tutorials/how_to_get_some_useful_stats.ipynb b/docs/source/tutorials/how_to_get_some_useful_stats.ipynb index 45915301..0688df2d 100644 --- a/docs/source/tutorials/how_to_get_some_useful_stats.ipynb +++ b/docs/source/tutorials/how_to_get_some_useful_stats.ipynb @@ -1877,7 +1877,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "But what if we dont want monthly coverage? We can then use the period variable to return whatever time period we want, whether that is 10-min (period='10min'), hourly (period='1H'), daily (period='1D'), weekly (period='1W') or yearly (period='1AS'). Here we have opted to return the yearly coverage." + "But what if we dont want monthly coverage? We can then use the period variable to return whatever time period we want, whether that is 10-min (period='10min'), hourly (period='1h'), daily (period='1D'), weekly (period='1W') or yearly (period='1YS'). Here we have opted to return the yearly coverage." ] }, { @@ -1964,7 +1964,7 @@ } ], "source": [ - "bw.coverage(data[anemometers],period='1AS')" + "bw.coverage(data[anemometers],period='1YS')" ] }, { diff --git a/docs/source/tutorials/how_to_transform_your_data.ipynb b/docs/source/tutorials/how_to_transform_your_data.ipynb index 39c18960..d45985c0 100644 --- a/docs/source/tutorials/how_to_transform_your_data.ipynb +++ b/docs/source/tutorials/how_to_transform_your_data.ipynb @@ -685,7 +685,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This function can be used to apply a window average to data, using a window which can be specified as any number of minutes ('min'), hours ('H'), days ('D'), weeks ('W'). '1M' and '1AS' can also used for monthly and annual averages respectively. The output is created within a new series. " + "This function can be used to apply a window average to data, using a window which can be specified as any number of minutes ('min'), hours ('h'), days ('D'), weeks ('W'). '1M' and '1YS' can also used for monthly and annual averages respectively. The output is created within a new series. " ] }, { @@ -1013,7 +1013,7 @@ ], "source": [ "# add 90 minutes to timestamps\n", - "data = bw.offset_timestamps(data, '1.5H')\n", + "data = bw.offset_timestamps(data, '1.5h')\n", "\n", "# print first timestamp\n", "print(data.index[0])" @@ -1041,7 +1041,7 @@ ], "source": [ "# subtract 2 hours from timestamps\n", - "data = bw.offset_timestamps(data, '-2H')\n", + "data = bw.offset_timestamps(data, '-2h')\n", "\n", "# print first timestamp\n", "print(data.index[0])" @@ -1332,7 +1332,7 @@ "outputs": [], "source": [ "# create new dataset with 6H added to the original timestamps\n", - "data_adj = bw.offset_timestamps(data, '6H')" + "data_adj = bw.offset_timestamps(data, '6h')" ] }, { diff --git a/setup.py b/setup.py index 595c573c..7e1c8e6d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def get_version(rel_path): 'gmaps>=0.9.0', 'colormap>=1.0.1', 'easydev>=0.10.0', - 'jsonschema>=4.17.3' + 'jsonschema>=4.17.3', + 'jinja2>= 3.0.0' ], classifiers=[ "Programming Language :: Python :: 3.6", diff --git a/tests/test_analyse.py b/tests/test_analyse.py index bc1f5159..1dde3149 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -139,9 +139,9 @@ def test_sector_ratio(): def test_basic_stats(): bw.basic_stats(DATA) bs2 = bw.basic_stats(DATA['Spd80mN']) - assert (bs2['count'][0] == 95180.0) and (round(bs2['mean'][0], 6) == 7.518636) and \ - (round(bs2['std'][0], 6) == 3.994552) and (round(bs2['min'][0], 3) == 0.215) and \ - (round(bs2['max'][0], 1) == 29.0) + assert (bs2['count'].iloc[0] == 95180.0) and (round(bs2['mean'].iloc[0], 6) == 7.518636) and \ + (round(bs2['std'].iloc[0], 6) == 3.994552) and (round(bs2['min'].iloc[0], 3) == 0.215) and \ + (round(bs2['max'].iloc[0], 1) == 29.0) def test_time_continuity_gaps(): @@ -209,9 +209,9 @@ def test_ti_twelve_by_24(): def test_coverage(): # hourly coverage - assert round(bw.coverage(DATA[['Spd80mN']], period='1H')[ + assert round(bw.coverage(DATA[['Spd80mN']], period='1h')[ '2016-01-09 17:00':'2016-01-09 17:30'].values[0][0], 5) == 0.83333 - assert round(bw.coverage(DATA.Spd80mN, period='1H')[ + assert round(bw.coverage(DATA.Spd80mN, period='1h')[ '2016-01-09 17:00':'2016-01-09 17:30'].values[0], 5) == 0.83333 # monthly_coverage assert round(bw.coverage(DATA.Spd80mN, period='1M')['2016-05-01'], 5) == 0.36537 diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 62964252..1708168a 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -102,7 +102,7 @@ def test_ordinary_least_squares(): assert round(correl.params['num_data_points'], 5) == correl_monthly_results_90_intercept['num_data_points'] # check hourly, checked against Excel - correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1H', + correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1h', coverage_threshold=1) correl.run() assert round(correl.params['slope'], 5) == correl_hourly_results['slope'] @@ -113,7 +113,7 @@ def test_ordinary_least_squares(): # check aggregation method correl_aggregation_results = {'slope': 5.98789, 'offset': -9.32585, 'r2': 0.9304, 'num_data_points': 12445} correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['T2M_degC'], DATA_CLND['T2m'], - averaging_prd='1H', coverage_threshold=1, + averaging_prd='1h', coverage_threshold=1, ref_aggregation_method='sum', target_aggregation_method='sum') correl.run() correl.plot() @@ -277,7 +277,7 @@ def test_synthesize(): for idx, row in pd.DataFrame(result_ord_lst_sq).iterrows(): # Comparing the first 6 digits to avoid issuing with floating point precision - assert str(row[0])[0:6] == str(synth.loc[idx][0])[0:6] + assert str(row.iloc[0])[0:6] == str(synth.loc[idx].iloc[0])[0:6] # Test the synthesise for when the ref_dir is given as input. correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['WS50m_m/s']['2016-03-02 00:00:00':], @@ -288,7 +288,7 @@ def test_synthesize(): synth = correl.synthesize() for idx, row in pd.DataFrame(result_ord_lst_sq_dir).iterrows(): - assert str(row[0]) == str(round(synth.loc[idx][0], 6)) + assert str(row.iloc[0]) == str(round(synth.loc[idx].iloc[0], 6)) # Test the synthesise when SpeedSort correlation is used. correl = bw.Correl.SpeedSort(MERRA2_NE['WS50m_m/s']['2016-03-02 00:00:00':'2017-03-02 00:00:00'], @@ -301,14 +301,15 @@ def test_synthesize(): for idx, row in pd.DataFrame(result_speed_sort).iterrows(): print(idx) - assert str(row[0]) == str(round(synth.loc[idx][0], 6)) + assert str(row.iloc[0]) == str(round(synth.loc[idx].iloc[0], 6)) # Test the synthesise when SpeedSort correlation is used using 10 min averaging period. data_test = DATA_CLND[['Spd80mN', 'Spd60mN', 'Dir78mS', 'Dir58mS']].copy() - data_test['Dir78mS']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan - data_test['Spd80mN']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan - data_test['Dir58mS']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan - data_test['Spd60mN']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan + data_test.loc['2016-01-09 17:10:00':'2016-01-09 17:50:00', 'Dir78mS'] = np.nan + data_test.loc['2016-01-09 17:10:00':'2016-01-09 17:50:00', 'Spd80mN'] = np.nan + data_test.loc['2016-01-09 17:50:00':'2016-01-10 19:10:00', 'Dir58mS'] = np.nan + data_test.loc['2016-01-09 17:50:00':'2016-01-10 19:10:00', 'Spd60mN'] = np.nan + ss_cor = bw.Correl.SpeedSort(data_test['Spd80mN'], data_test['Dir78mS'], data_test['Spd60mN'], data_test['Dir58mS'], averaging_prd='10min') ss_cor.run() @@ -370,7 +371,7 @@ def test_orthogonal_least_squares(): assert round(correl.params['num_data_points'], 5) == correl_monthly_results['num_data_points'] # check hourly - correl = bw.Correl.OrthogonalLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1H', + correl = bw.Correl.OrthogonalLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1h', coverage_threshold=1) correl.run() assert round(correl.params['slope'], 5) == correl_hourly_results['slope'] @@ -381,7 +382,7 @@ def test_orthogonal_least_squares(): # check aggregation method correl_aggregation_results = {'slope': 6.42434, 'offset': -12.8301, 'r2': 0.9255, 'num_data_points': 12445} correl = bw.Correl.OrthogonalLeastSquares(MERRA2_NE['T2M_degC'], DATA_CLND['T2m'], - averaging_prd='1H', coverage_threshold=1, + averaging_prd='1h', coverage_threshold=1, ref_aggregation_method='sum', target_aggregation_method='sum') correl.run() assert round(correl.params['slope'], 5) == correl_aggregation_results['slope'] @@ -403,7 +404,7 @@ def test_multiple_linear_regression(): # check aggregation method correl_aggregation_results = {'slope': [5.51666, 0.54769], 'offset': -10.44818, 'num_data_pts': 12445} correl = bw.Correl.MultipleLinearRegression([MERRA2_NE['T2M_degC'], MERRA2_NW['T2M_degC']], DATA_CLND['T2m'], - averaging_prd='1H', coverage_threshold=1, + averaging_prd='1h', coverage_threshold=1, ref_aggregation_method='sum', target_aggregation_method='sum') correl.run() for idx, slope in enumerate(correl.params['slope']): @@ -570,7 +571,7 @@ def test_speed_sort(): } ss_cor = bw.Correl.SpeedSort(MERRA2_NE['WS50m_m/s'], MERRA2_NE['WD50m_deg'], DATA_CLND['Spd80mN'], DATA_CLND['Dir78mS'], - averaging_prd='1H') + averaging_prd='1h') ss_cor.run() assert ss_cor.params['overall_average_veer'] == result['overall_average_veer'] diff --git a/tests/test_shear.py b/tests/test_shear.py index 39a284d8..bf083c69 100644 --- a/tests/test_shear.py +++ b/tests/test_shear.py @@ -136,8 +136,8 @@ def test_time_of_day(): # Test attributes - assert round(shear_by_tod_power_law2.alpha.mean()[0], 4) == 0.1473 - assert round(shear_by_tod_log_law2.roughness.mean()[0], 4) == 0.1450 + assert round(shear_by_tod_power_law2.alpha.mean().iloc[0], 4) == 0.1473 + assert round(shear_by_tod_log_law2.roughness.mean().iloc[0], 4) == 0.1450 # Test apply assert (round(DATA['Spd80mN']['2017-11-23 10:10:00':'2017-11-23 10:40:00'] * ( @@ -221,12 +221,12 @@ def test_time_series(): shear_by_ts_power_law.apply(DATA['Spd80mN'], 40, 60) shear_by_ts_log_law.apply(DATA['Spd80mN'], 40, 60) - DATA['Spd80mN'].iloc[0] = 2 + DATA.loc[DATA.index[0], 'Spd80mN'] = 2 shear_ts = bw.Shear.TimeSeries(anemometers, heights) alpha = shear_ts.alpha assert pd.isna(alpha.iloc[0]) - DATA['Spd80mN'].iloc[0] = np.nan + DATA.loc[DATA.index[0], 'Spd80mN'] = np.nan shear_ts = bw.Shear.TimeSeries(anemometers, heights, calc_method='log_law') roughness = shear_ts.roughness assert pd.isna(roughness.iloc[0]) diff --git a/tests/test_transform.py b/tests/test_transform.py index 932442d5..fe9b1c2b 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -7,6 +7,7 @@ import io import sys import re +import warnings from brightwind.transform.transform import ( _check_vertical_profiler_properties_overlap as check_vertical_profiler_properties_overlap @@ -292,13 +293,13 @@ def test_check_vertical_profiler_properties_not_overlap(): def test_freq_str_to_dateoffset(): # Excluding monthly periods and above as it will depend on which month or year periods = ['1S', '1min', '5min', '10min', '15min', - '1H', '3H', '6H', '1D', '7D', + '1h', '3h', '6h', '1D', '7D', '1W', '2W', '1MS', '1M', '3M', '6MS', - '1AS', '1A', '3A'] + '1YS'] results = [1.0, 60.0, 300.0, 600.0, 900.0, 3600.0, 10800.0, 21600.0, 86400.0, 604800.0, 604800.0, 1209600.0, 2678400.0, 2678400.0, 7862400.0, 15724800.0, - 31622400.0, 31622400.0, 94694400.0] + 31622400.0] for idx, period in enumerate(periods): if type(bw.transform.transform._freq_str_to_dateoffset(period)) == pd.DateOffset: @@ -308,18 +309,62 @@ def test_freq_str_to_dateoffset(): ).total_seconds() == results[idx] # Check that data frequency is returned as a DateOffset. - assert type(bw.transform.transform._freq_str_to_dateoffset(period)) == pd.DateOffset + assert isinstance(bw.transform.transform._freq_str_to_dateoffset(period), pd.DateOffset) + + +def test_freq_str_to_dateoffset_deprecation_warning(): + """Test frequency string conversion and check deprecation warnings for old formats.""" + ref_date = pd.Timestamp('2020-01-01') + + periods = [ + '1H', '3H', '6H','1T', '30T', '1S', '1AS', '1A', '3A' + ] + results = [ + 3600.0, 10800.0, 21600.0, 60.0, 1800.0, 1.0, 31622400.0, 31622400.0, 94694400.0 + ] + + # Reset warning flags to test them + bw.transform.transform._warned_h = False + bw.transform.transform._warned_t = False + bw.transform.transform._warned_s = False + bw.transform.transform._warned_a = False + bw.transform.transform._warned_as = False + + for idx, period in enumerate(periods): + # All these deprecated formats should raise DeprecationWarning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = bw.transform.transform._freq_str_to_dateoffset(period) + + # Check warning was raised on first use of each suffix + if period in ['1H', '1T', '1S', '1AS', '1A']: + assert len(w) == 1, f"Expected warning for deprecated format '{period}'" + assert issubclass(w[0].category, DeprecationWarning) + + # Check that data frequency is returned as a DateOffset + assert isinstance(result, pd.DateOffset) + + # Verify the time delta + if type(result) == pd.DateOffset: + assert (ref_date + result - ref_date).total_seconds() == results[idx] def test_round_timestamp_down_to_averaging_prd(): timestamp = pd.Timestamp('2016-01-09 11:21:11') - avg_periods = ['10min', '15min', '1H', '3H', '6H', '1D', '7D', '1W', '1MS', '1AS'] + avg_periods = ['10min', '15min', '1h', '3h', '6h', '1D', '7D', '1W', '1MS', '1YS'] avg_period_start_timestamps = ['2016-1-9 11:20:00', '2016-1-9 11:15:00', '2016-1-9 11:00:00', '2016-1-9 9:00:00', '2016-1-9 6:00:00', '2016-1-9', '2016-1-9', '2016-1-9', '2016-1', '2016'] for idx, avg_period in enumerate(avg_periods): assert avg_period_start_timestamps[idx] == \ bw.transform.transform._round_timestamp_down_to_averaging_prd(timestamp, avg_period) + avg_periods = ['1H', '3H', '6H', '1AS'] + avg_period_start_timestamps = [ + '2016-1-9 11:00:00', '2016-1-9 9:00:00', '2016-1-9 6:00:00', '2016' + ] + for idx, avg_period in enumerate(avg_periods): + assert avg_period_start_timestamps[idx] == \ + bw.transform.transform._round_timestamp_down_to_averaging_prd(timestamp, avg_period) def test_get_data_resolution(): @@ -328,7 +373,7 @@ def test_get_data_resolution(): series1 = DATA['Spd80mS'].index assert bw.transform.transform._get_data_resolution(series1).kwds == {'minutes': 10} - series2 = pd.date_range('2010-01-01', periods=150, freq='H') + series2 = pd.date_range('2010-01-01', periods=150, freq='h') assert bw.transform.transform._get_data_resolution(series2).kwds == {'hours': 1} series2 = pd.date_range('2010-01-01', periods=150, freq='D') @@ -340,9 +385,19 @@ def test_get_data_resolution(): series1 = bw.average_data_by_period(DATA['Spd80mN'], period='1M', coverage_threshold=0, return_coverage=False) assert bw.transform.transform._get_data_resolution(series1.index).kwds == {'months': 1} + series1 = bw.average_data_by_period(DATA['Spd80mN'], period='1YS', coverage_threshold=0, return_coverage=False) + assert bw.transform.transform._get_data_resolution(series1.index).kwds == {'years': 1} + series1 = bw.average_data_by_period(DATA['Spd80mN'], period='1AS', coverage_threshold=0, return_coverage=False) assert bw.transform.transform._get_data_resolution(series1.index).kwds == {'years': 1} + # hourly series with one instance where difference between adjacent timestamps is 10 min + series3 = pd.date_range('2010-04-15', '2010-05-01', freq='h').union(pd.date_range('2010-05-01 00:10:00', periods=20, + freq='h')) + with warnings.catch_warnings(record=True) as w: + assert bw.transform.transform._get_data_resolution(series3).kwds == {'hours': 1} + assert len(w) == 1 + # hourly series with one instance where difference between adjacent timestamps is 10 min series3 = pd.date_range('2010-04-15', '2010-05-01', freq='H').union(pd.date_range('2010-05-01 00:10:00', periods=20, freq='H')) @@ -449,9 +504,14 @@ def test_offset_timestamps(): assert (op.loc['2016-01-12 00:00:00'] == series1.Spd60mN.loc['2016-01-11 23:30:00']).all() assert (op.loc['2016-01-12 00:30:00'] == series1.Spd60mN.loc['2016-01-12 00:30:00']).all() + assert bw.offset_timestamps(DATA.index[0], offset='4h') == pd.Timestamp('2016-01-09 19:30:00') + assert bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5h' + ) == datetime.datetime(2016, 2, 1, 3, 50) + assert bw.offset_timestamps(DATA.index[0], offset='4H') == pd.Timestamp('2016-01-09 19:30:00') assert bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5H' ) == datetime.datetime(2016, 2, 1, 3, 50) + assert bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5h') == datetime.datetime(2016, 1, 31, 19, 0) assert bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5H') == datetime.datetime(2016, 1, 31, 19, 0) assert bw.offset_timestamps(datetime.time(0, 20), offset='30min') == datetime.time(0, 50) @@ -541,46 +601,48 @@ def test_average_wdirs(): for i, j in zip(avg_wdirs, expected_result): assert i == j - -def dummy_data_frame(start_date='2016-01-01T00:00:00', end_date='2016-12-31T11:59:59'): +@pytest.fixture +def dummy_data(): """ - Returns a DataFrame with wind speed equal to the month of the year, i.e. In January, wind speed = 1 m/s. - For use in testing. - - :param start_date: Start date Timestamp, i.e. first index in the DataFrame - :type start_date: Timestamp as a string in the form YYYY-MM-DDTHH:MM:SS' - :param end_date: End date Timestamp, i.e. last index in the DataFrame - :type end_date: Timestamp as a string in the form YYYY-MM-DDTHH:MM:SS' - :return: pandas.DataFrame + Fixture that returns a DataFrame with wind speed equal to the month of the year. + Wind speed in January = 1 m/s, February = 2 m/s, etc. """ - - date_times = {'Timestamp': pd.date_range(start_date, end_date, freq='10T')} - - dummy_wind_speeds = [] - dummy_wdirs = [] - - for i, vals in enumerate(date_times['Timestamp']): - # get list of each month for each date entry as dummy windspeeds - dummy_wind_speeds.append(vals.month) - dummy_wdirs.append((vals.month - 1) * 30) - - dummy_wind_speeds_df = pd.DataFrame({'wspd': dummy_wind_speeds, 'wdir': dummy_wdirs}, index=date_times['Timestamp']) + start_date = '2016-01-01T00:00:00' + end_date = '2016-12-31T11:59:59' + + date_times = pd.date_range(start_date, end_date, freq='10min') + + # Vectorized approach - compatible with all pandas versions + dummy_wind_speeds = date_times.month.values + dummy_wdirs = (date_times.month.values - 1) * 30 + + dummy_wind_speeds_df = pd.DataFrame( + {'wspd': dummy_wind_speeds, 'wdir': dummy_wdirs}, + index=date_times + ) dummy_wind_speeds_df.index.name = 'Timestamp' - + return dummy_wind_speeds_df -def test_average_data_by_period(): +def test_average_data_by_period(dummy_data): + bw.average_data_by_period(DATA[['Spd80mN']], period='1h') bw.average_data_by_period(DATA[['Spd80mN']], period='1H') # hourly averages + bw.average_data_by_period(DATA.Spd80mN, period='1h') bw.average_data_by_period(DATA.Spd80mN, period='1H') # hourly average with coverage filtering + bw.average_data_by_period(DATA.Spd80mN, period='1h', coverage_threshold=0.9) + bw.average_data_by_period(DATA.Spd80mN, period='1h', coverage_threshold=1) bw.average_data_by_period(DATA.Spd80mN, period='1H', coverage_threshold=0.9) bw.average_data_by_period(DATA.Spd80mN, period='1H', coverage_threshold=1) # return coverage with filtering + bw.average_data_by_period(DATA.Spd80mN, period='1h', coverage_threshold=0.9, + return_coverage=True) bw.average_data_by_period(DATA.Spd80mN, period='1H', coverage_threshold=0.9, return_coverage=True) # return coverage without filtering + bw.average_data_by_period(DATA.Spd80mN, period='1h', return_coverage=True) bw.average_data_by_period(DATA.Spd80mN, period='1H', return_coverage=True) # monthly averages @@ -608,7 +670,6 @@ def test_average_data_by_period(): assert str(except_info.value) == "The time period specified is less than the temporal resolution of the data. " \ "For example, hourly data should not be averaged to 10 minute data." - dummy_data = dummy_data_frame() average_monthly_speed = bw.average_data_by_period(dummy_data.wspd, period='1M') # test average wind speed for each month for i in range(0, 11): @@ -641,6 +702,8 @@ def test_average_data_by_period(): assert average_monthly_speed[1].count().wspd_Coverage == 12 # the returned coverage has 12 months # test average annual wind speed + average_annual_speed = bw.average_data_by_period(dummy_data.wspd, period='1YS') + assert round(average_annual_speed.iloc[0].item(), 3) == 6.506 average_annual_speed = bw.average_data_by_period(dummy_data.wspd, period='1AS') assert round(average_annual_speed.iloc[0].item(), 3) == 6.506 # average DATA to monthly @@ -686,8 +749,7 @@ def test_average_data_by_period(): data_monthly, coverage_monthly = bw.average_data_by_period(data_test, period='1M', wdir_column_names='Dir78mS', return_coverage=True, data_resolution=pd.DateOffset(minutes=10)) - table_count = data_test.resample('1MS', axis=0, closed='left', label='left', - convention='start', kind='timestamp').count() + table_count = data_test.resample('1MS', closed='left', label='left').count() assert (table_count['Dir78mS']['2016-01-01'] / (31 * 24 * 6) - coverage_monthly['Dir78mS_Coverage']['2016-01-01'] ) < 1e-5 assert (table_count['Spd80mN']['2016-01-01'] / (31 * 24 * 6) - coverage_monthly['Spd80mN_Coverage']['2016-01-01'] From 3c72734c30ce4222cfbd3ade12705b6bb56ad779 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Tue, 16 Dec 2025 13:06:29 +0000 Subject: [PATCH 24/35] iss #556 add deprecated warning to BrightData (#564) * iss #556 add deprecated warning to BrightData * iss #556 fix text * iss #556 added additional deprecated notices while i was at it --- CHANGELOG.md | 6 +++++- brightwind/load/load.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30ed7d16..cff8f6a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ Additional labels for pre-release and build metadata are available as extensions 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) 1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) - ### Bug Fixes 1. @@ -34,6 +33,11 @@ Additional labels for pre-release and build metadata are available as extensions 1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) 1. Updated `offset_timestamps()`to include last timestamp when date_to is unspecified, so that offset is applied to the entire record if date_to not specified. ([#504](https://github.com/brightwind-dev/brightwind/issues/504)) +### Deprecated +1. `LoadBrightdata()` is deprecated and will be removed in version 3.0. Please use `LoadBrightHub()` instead to continue accessing reanalysis data. +2. `LoadBrightHub()` authentication using email and password is deprecated and will be removed in v3.0. Please migrate to API key authentication. Create and manage API keys at: https://brighthub.io/account-settings/settings. +3. `specific_gas_constant` argument of `calc_air_density()` is deprecated and will be removed in v3.0 because the updated method depends on the gas constants for dry air and water vapour, making this argument redundant. +4. The scaling of air density to height within `calc_air_density()` is deprecated and will be removed in v3.0. Users should use `scale_air_density_to_height()` separately instead. ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 7f02f700..bcf5a8e6 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -447,6 +447,12 @@ def _append_files_together(source_folder, assembled_file_name, file_type, append class LoadBrightdata: """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + LoadBrightdata allows you to pull meta data and timeseries data of reanalysis datasets from brightdata. This is a fast way to get access to the available reanalysis datasets. @@ -497,6 +503,15 @@ def _get_brightdata(sub_uri, **query_params): :param query_params: dictionary of the query parameters to be sent :return: List(Node) """ + + warnings.warn( + "BrightDATA was deprecated in v2.4.0 and will be removed in v3.0.0. " + "Support for reanalysis data has moved to BrightHub. " + "Please use LoadBrightHub instead of LoadBrightdata to continue accessing reanalysis data.", + DeprecationWarning, + stacklevel=3 + ) + username = utils.get_environment_variable('BRIGHTDATA_USERNAME') password = utils.get_environment_variable('BRIGHTDATA_PASSWORD') @@ -548,6 +563,12 @@ def _parse_variables(variables_list): @staticmethod def timeseries(dataset, lat, long, nearest=None, from_date=None, to_date=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Retrieve timeseries datasets available from brightdata. Returns a list of Node objects in order of closest distance to the requested lat, long. @@ -632,6 +653,12 @@ def timeseries(dataset, lat, long, nearest=None, from_date=None, to_date=None, v @staticmethod def monthly_means(dataset, lat, long, nearest=None, from_date=None, to_date=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Retrieve monthly means from brightdata datasets such as merra2 and era5. Returns a list of Node objects in order of closest distance to the requested lat, long. Monthly coverage is also returned in the data. @@ -713,6 +740,12 @@ def monthly_means(dataset, lat, long, nearest=None, from_date=None, to_date=None @staticmethod def momm(dataset, lat, long, nearest=None, from_date=None, to_date=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Retrieve the mean of monthly means from brightdata datasets such as merra2 and era5. Returns a list of Node objects in order of closest distance to the requested lat, long. @@ -796,6 +829,12 @@ def momm(dataset, lat, long, nearest=None, from_date=None, to_date=None, variabl def monthly_norms(dataset, lat, long, nearest=None, from_date=None, to_date=None, ref_from_date=None, ref_to_date=None, ref_no_years=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Return the monthly mean wind speeds normalised to a specific reference period. The reference period can be between two specific dates or it could be a number of rolling years preceding each month of interest. From 8955dae89dcdd6d66e35f46c851f20852e457cfd Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Tue, 16 Dec 2025 15:03:53 +0000 Subject: [PATCH 25/35] iss #561 renamed series with _scaled (#563) --- brightwind/transform/scale.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/brightwind/transform/scale.py b/brightwind/transform/scale.py index 7bf903ea..108026c6 100644 --- a/brightwind/transform/scale.py +++ b/brightwind/transform/scale.py @@ -221,18 +221,21 @@ def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], ref_height_m=2, target_height_m=10) - Timestamp - 2016-01-09 17:10:00 1.186780 - 2016-01-09 17:20:00 1.187175 - 2016-01-09 17:30:00 1.187747 - 2016-01-09 17:40:00 1.185950 - 2016-01-09 17:50:00 1.186301 - 2016-01-09 18:00:00 1.185686 - dtype: float64 + # Timestamp + # 2016-01-09 17:10:00 1.186780 + # 2016-01-09 17:20:00 1.187175 + # 2016-01-09 17:30:00 1.187747 + # 2016-01-09 17:40:00 1.185950 + # 2016-01-09 17:50:00 1.186301 + # 2016-01-09 18:00:00 1.185686 + # Name: air_density_scaled, dtype: float64 """ scaled_air_density = linear_transform(x_target=target_height_m, x_ref=ref_height_m, y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) + if isinstance(scaled_air_density, pd.Series): + air_density_series_name = ref_air_density_kg_m3.name if ref_air_density_kg_m3.name else "air_density" + scaled_air_density.name = f"{air_density_series_name}_scaled" return scaled_air_density @@ -298,11 +301,14 @@ def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series] # 2016-01-09 17:40:00 0.735 # 2016-01-09 17:50:00 0.654 # 2016-01-09 18:00:00 0.796 - # Name: T2m, dtype: float64 + # Name: T2m_scaled, dtype: float64 """ scaled_air_temp = linear_transform(x_target=target_height_m, x_ref=ref_height_m, y_ref=ref_air_temperature, slope=lapse_rate_deg_m) + if isinstance(scaled_air_temp, pd.Series): + air_temp_series_name = ref_air_temperature.name if ref_air_temperature.name else "air_temperature" + scaled_air_temp.name = f"{air_temp_series_name}_scaled" return scaled_air_temp @@ -360,7 +366,7 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], # 2016-01-09 17:40:00 932.07 # 2016-01-09 17:50:00 932.07 # 2016-01-09 18:00:00 932.07 - # dtype: float64 + # Name: P2m_scaled, dtype: float64 """ @@ -386,5 +392,7 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) ) ** (-g / (L * R))) - + if isinstance(scaled_air_pressure_hPa, pd.Series): + air_pressure_series_name = ref_air_pressure_hPa.name if ref_air_pressure_hPa.name else "air_pressure" + scaled_air_pressure_hPa.name = f"{air_pressure_series_name}_scaled" return scaled_air_pressure_hPa From c6eaa43b7ef7ebdd869835f3650f985d17d2573c Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Wed, 17 Dec 2025 10:13:01 +0000 Subject: [PATCH 26/35] [Iss566] Handle dataframes in calc_air_density (#567) * iss #566 raise error if dataframe with more than one column provided * iss #566 remove dataframe type and edit tests --- brightwind/analyse/analyse.py | 26 +++++++++++++------------- tests/test_analyse.py | 10 ++-------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index bce9e967..9c10a7a9 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -2042,13 +2042,13 @@ def sector_ratio(wspd_1, wspd_2, wdir, sectors=72, min_wspd=3, direction_bin_arr return fig -def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], - pressure: Union[float, pd.Series, pd.DataFrame], +def calc_air_density(temperature: Union[float, pd.Series], + pressure: Union[float, pd.Series], elevation_ref: Union[float, int] = None, elevation_site: Union[float, int] = None, lapse_rate: float = AIR_DENSITY_LAPSE_RATE, specific_gas_constant: float = 286.9, - rel_humidity_percent: Union[float, pd.Series, pd.DataFrame] = None + rel_humidity_percent: Union[float, pd.Series] = None ) -> Union[float, pd.Series]: """ Calculates air density for a given air temperature (temperature), air pressure (pressure) @@ -2072,10 +2072,10 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], `scale_air_density_to_height()` separately instead. :param temperature: Air temperature values in degree Celsius - :type temperature: float or pandas.Series or pandas.DataFrame + :type temperature: float or pandas.Series :param pressure: Air pressure values in hectopascal, hPa or millibar, mbar (Note 1hPa = 1mbar) (1013.25 hPa = 101,325 Pa = 101.325 kPa = 1 atm = 1013.25 mbar) - :type pressure: float or pandas.Series or pandas.DataFrame + :type pressure: float or pandas.Series :param elevation_ref: Elevation, in meters, of the reference air density location (and air temperature, air pressure, relative humidity measurements used for the calculation). :type elevation_ref: float or int @@ -2094,10 +2094,10 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], :type specific_gas_constant: float :param rel_humidity_percent: Relative humidity values as a percentage. Default is None. If None, the air density calculation ignores humidity. - :type rel_humidity_percent: float or pandas.Series or pandas.DataFrame + :type rel_humidity_percent: float or pandas.Series :return: Air density in kg/m^3. Output type depends on type(temperature), type(pressure), type(rel_humidity_percent) provided. If all inputs are float, output is float. - If any input is pandas.Series or pandas.DataFrame, output is pandas.Series. + If any input is pandas.Series, output is pandas.Series. :rtype: float or pandas.Series **Example usage** @@ -2160,18 +2160,18 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], # 1.198 """ - # Convert DataFrame inputs to Series + # Raise errors if any inputs are DataFrames if isinstance(temperature, pd.DataFrame): - temperature = _convert_df_to_series(temperature).copy() + raise ValueError("temperature cannot be provided as a DataFrame. Provide as float or pandas Series.") if isinstance(pressure, pd.DataFrame): - pressure = _convert_df_to_series(pressure).copy() + raise ValueError("pressure cannot be provided as a DataFrame. Provide as float or pandas Series.") if isinstance(rel_humidity_percent, pd.DataFrame): - rel_humidity_percent = _convert_df_to_series(rel_humidity_percent).copy() - + raise ValueError("rel_humidity_percent cannot be provided as a DataFrame. Provide as float or pandas Series.") + # Check dimensions of temperature, pressure and rel_humidity_percent if not float or int if isinstance(temperature, pd.Series) and (isinstance(pressure, pd.Series)): if len(temperature) != len(pressure): - raise ValueError("temperature and pressure must have the same dimensions.") + raise ValueError("temperature and pressure must have the same dimensions.") if isinstance(rel_humidity_percent, pd.Series): if len(temperature) != len(rel_humidity_percent): raise ValueError("temperature, pressure and rel_humidity_percent must have the same dimensions.") diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 1dde3149..e61e1d3b 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -638,12 +638,6 @@ def test_ti_by_sector(): def test_calc_air_density(): - - # test DataFrame input - assert (bw.calc_air_density(DATA[['T2m']], DATA[['P2m']]).dropna().values == - bw.calc_air_density(DATA.T2m, DATA.P2m).dropna().values).all() - assert (round(bw.calc_air_density(pd.Series([15, 12.5, -5, 23]), pd.Series([1013, 990, 1020, 900])), 6 - ) == pd.Series([1.225350, 1.208010, 1.325842, 1.059254])).all() # test Series inputs assert list(round(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5), 6).values @@ -684,10 +678,10 @@ def test_calc_air_density(): bw.calc_air_density(15, 1013, elevation_ref=200) assert str(except_info.value) == "Specify value of elevation_site (float or int) when elevation_ref is provided." with pytest.raises(ValueError) as except_info: - bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) + bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) assert str(except_info.value) == "temperature and pressure must have the same dimensions." with pytest.raises(ValueError) as except_info: - bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) + bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) assert str(except_info.value) == "temperature, pressure and rel_humidity_percent must have the same dimensions." From a1713c61c977051f25b8c6d2dae72328d6c4c489 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Wed, 17 Dec 2025 10:27:59 +0000 Subject: [PATCH 27/35] prepare for v2.4.0 release (#565) --- CHANGELOG.md | 14 ++++++++------ brightwind/__init__.py | 23 ++++++++++++++++++++++- setup.py | 4 ++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cff8f6a6..7013f90c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,7 @@ Additional labels for pre-release and build metadata are available as extensions --- -## [2.4.0-dev] -1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) -1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) - -### Bug Fixes -1. +## [2.4.0] ### New Features and Enhancements 1. Added `scale_air_pressure_to_height()` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height. ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) @@ -38,6 +33,13 @@ Additional labels for pre-release and build metadata are available as extensions 2. `LoadBrightHub()` authentication using email and password is deprecated and will be removed in v3.0. Please migrate to API key authentication. Create and manage API keys at: https://brighthub.io/account-settings/settings. 3. `specific_gas_constant` argument of `calc_air_density()` is deprecated and will be removed in v3.0 because the updated method depends on the gas constants for dry air and water vapour, making this argument redundant. 4. The scaling of air density to height within `calc_air_density()` is deprecated and will be removed in v3.0. Users should use `scale_air_density_to_height()` separately instead. +5. Support for **Python ≤ 3.10** is deprecated and will be dropped in v3.0.0. Users should upgrade to Python 3.11 or newer. +6. Support for **Pandas ≤ 2.2** is deprecated and will be dropped in v3.0.0. Users should upgrade to Pandas 2.3 or newer. + +### Bug Fixes +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) + ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 5814a0f6..d865d7f5 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -11,6 +11,27 @@ from .utils.gis import * # from .utils.utils import * +import sys +import warnings +from packaging.version import Version +import pandas as pd + +if sys.version_info < (3, 11): + warnings.warn( + "Support for Python 3.10 is deprecated and will be removed in v3.0.0. " + "Please upgrade to Python 3.11 or newer.", + FutureWarning, + stacklevel=2, + ) + +if Version(pd.__version__) <= Version("2.2"): + warnings.warn( + "Support for pandas ≤ 2.2 is deprecated and will be removed in v3.0.0. " + "Please upgrade to pandas 2.3 or newer.", + FutureWarning, + stacklevel=2, + ) + __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.4.0-dev' +__version__ = '2.4.0' diff --git a/setup.py b/setup.py index 7e1c8e6d..3c4fe9c7 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def get_version(rel_path): setup( name='brightwind', - # Update version number here, below and in _init_.py: + # Update version number below and in _init_.py: version=get_version("brightwind/__init__.py"), packages=['brightwind', 'brightwind.load', 'brightwind.utils', 'brightwind.export', 'brightwind.analyse', 'brightwind.transform', 'brightwind.demo_datasets'], @@ -34,7 +34,7 @@ def get_version(rel_path): }, url='https://github.com/brightwind-dev/brightwind.git', # UPDATE VERSION NUMBER HERE: - download_url='https://github.com/brightwind-dev/brightwind/archive/v2.3.0.tar.gz', + download_url='https://github.com/brightwind-dev/brightwind/archive/v2.4.0.tar.gz', license='MIT', author='Stephen Holleran of BrightWind Ltd', author_email='stephen@brightwindanalysis.com', From a634595aaac17a578b54917322a287eabcc3bd86 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Tue, 10 Feb 2026 12:34:32 +0000 Subject: [PATCH 28/35] [Iss539] Add alternative air density method (SR5) (#549) * iss #544 moved scaling functions to transform/scale.py * iss #539 added several functions for calculating air density, vapour pressure and saturation vapour pressure * iss #539 added tests * iss #539 updated changelog * iss #539 minor reformatting * iss #539 updated calc_air_density to handle three different calculation methods * iss #539 added examples, fixed formatting * iss #539 fixed formatting * iss #539 updated tests * iss #539 updated changelog * iss #539 moved constants to utils/constants.py * iss #539 fixed typo R_w * iss #539 updated to use global constant value for R_w * iss #539 added references to docstring of bw.calc_air_density * iss #539 tidied check dew point < temp * iss #539 updated assert type function name, docstring and added tests * iss #539 rewordings and format fixes * iss #539 moved input type check from utils to be a hidden function in analyse.py * iss #539 update error msg wording * iss #539 add function to assert series indexes match * iss #539 added check for series index match * iss #539 formatting * iss #539 reduced checking code and automatic PEP8 fix pycharm * Update CHANGELOG.md * iss #539 docstring edits * iss #539 more docstring edits --------- Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 10 +- brightwind/analyse/analyse.py | 674 ++++++++++++++++++++++++++++------ brightwind/transform/scale.py | 15 +- brightwind/utils/constants.py | 21 ++ brightwind/utils/utils.py | 5 - tests/test_analyse.py | 94 ++++- tests/test_utils.py | 1 + 7 files changed, 670 insertions(+), 150 deletions(-) create mode 100644 brightwind/utils/constants.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7013f90c..65faf3fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Given a version number MAJOR.MINOR.PATCH, increment the: Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. --- +## [2.5.0] +### New Features and Enhancements +1. Updated `calc_air_density` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). + ## [2.4.0] @@ -25,8 +29,8 @@ Additional labels for pre-release and build metadata are available as extensions 7. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). 8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. -1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) -1. Updated `offset_timestamps()`to include last timestamp when date_to is unspecified, so that offset is applied to the entire record if date_to not specified. ([#504](https://github.com/brightwind-dev/brightwind/issues/504)) +9. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) +10. Updated `offset_timestamps()`to include last timestamp when date_to is unspecified, so that offset is applied to the entire record if date_to not specified. ([#504](https://github.com/brightwind-dev/brightwind/issues/504)) ### Deprecated 1. `LoadBrightdata()` is deprecated and will be removed in version 3.0. Please use `LoadBrightHub()` instead to continue accessing reanalysis data. @@ -38,7 +42,7 @@ Additional labels for pre-release and build metadata are available as extensions ### Bug Fixes 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) -1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) +2. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 9c10a7a9..e9308e7c 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -1,12 +1,17 @@ import pandas as pd import numpy as np -from typing import Optional +from typing import Optional, Any from brightwind.transform import transform as tf from brightwind.utils import utils from brightwind.analyse import plot as bw_plt from brightwind.transform.scale import scale_air_density_to_height -from brightwind.utils.utils import _convert_df_to_series -from brightwind.utils.utils import validate_coverage_threshold +from brightwind.utils.utils import _convert_df_to_series, validate_coverage_threshold +from brightwind.utils.constants import ( + GAS_CONST_DRY_AIR, + GAS_CONST_WATER, + AIR_DENSITY_LAPSE_RATE, + DEGREES_CELSIUS_TO_KELVIN, +) from brightwind.export.export import _calc_mean_speed_of_freq_tab import matplotlib.pyplot as plt import warnings @@ -31,11 +36,6 @@ 'sector_ratio', 'calc_air_density'] -# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere -GAS_CONST_DRY_AIR = 287.05 - -# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) -AIR_DENSITY_LAPSE_RATE = -0.113 def dist_matrix(var_series, x_series, y_series, num_bins_x=None, num_bins_y=None, @@ -128,9 +128,9 @@ def custom_agg(x): var_label = aggregation_method.capitalize() + ' of ' + var_series.name var_series.name = var_label if x_series.name == var_series.name: - x_series.name = x_series.name+"_binned" + x_series.name = x_series.name + "_binned" if y_series.name == var_series.name: - y_series.name = y_series.name+"_binned" + y_series.name = y_series.name + "_binned" if num_bins_x is None and x_bins is None: x_bins = np.arange(int(np.floor(x_series.min())), int(np.ceil(x_series.max()) + 1 + (x_series.max() % 1 == 0)), @@ -185,7 +185,7 @@ def calc_target_value_by_linear_model(ref_value: float, slope: float, offset: fl """ :rtype: np.float64 """ - return (ref_value*slope) + offset + return (ref_value * slope) + offset def monthly_means(data, return_data=False, return_coverage=False, ylabel='Wind speed [m/s]', data_resolution=None, @@ -269,15 +269,15 @@ def monthly_means(data, return_data=False, return_coverage=False, ylabel='Wind s return bw_plt.plot_monthly_means( df, ylbl=ylabel, legend=legend, external_legend=external_legend, show_grid=show_grid, xtick_delta=xtick_delta - ), df + ), df if return_coverage: return bw_plt.plot_monthly_means( df, covrg, ylbl=ylabel, legend=legend, external_legend=external_legend, show_grid=show_grid, xtick_delta=xtick_delta - ), pd.concat([df, covrg], axis=1) + ), pd.concat([df, covrg], axis=1) return bw_plt.plot_monthly_means( df, ylbl=ylabel, legend=legend, external_legend=external_legend, show_grid=show_grid, xtick_delta=xtick_delta - ) + ) def _filter_out_months_based_on_coverage_threshold(var_series, monthly_coverage, coverage_threshold, analysis_type, @@ -548,9 +548,9 @@ def _get_direction_bin_labels(sectors, direction_bins, zero_centred=True): mapper = dict() for i, lower_bound in enumerate(direction_bins[:sectors]): if i == 0 and zero_centred: - mapper[i+1] = '{0}-{1}'.format(direction_bins[-2], direction_bins[1]) + mapper[i + 1] = '{0}-{1}'.format(direction_bins[-2], direction_bins[1]) else: - mapper[i+1] = '{0}-{1}'.format(lower_bound, direction_bins[i+1]) + mapper[i + 1] = '{0}-{1}'.format(lower_bound, direction_bins[i + 1]) return mapper.values() @@ -561,7 +561,7 @@ def _map_direction_bin(wdir, bins, sectors): else: kwargs['right'] = False bin_num = np.digitize([wdir], bins, **kwargs)[0] - if bin_num == sectors+1: + if bin_num == sectors + 1: bin_num = 1 return bin_num @@ -607,7 +607,7 @@ def _derive_distribution(var_to_bin, var_to_bin_against, bins=None, aggregation_ if var_to_bin_against.empty or var_to_bin_against.isnull().all() or (var_to_bin_against == np.inf).all(): raise ValueError(('Cannot derive distribution with respect to {0} as this is either an empty pandas.Series ' + 'or contains only NaN or Inf values.').format(var_to_bin_against.name)) - + var_to_bin = var_to_bin.replace([np.inf, -np.inf], np.nan).dropna() var_to_bin_against = var_to_bin_against.replace([np.inf, -np.inf], np.nan).dropna() @@ -784,7 +784,7 @@ def dist_of_wind_speed(wspd, max_speed=30, max_y_value=None, return_data=False): freq_dist_plot, freq_dist = bw.dist_of_wind_speed(data.Spd80mN, return_data=True) """ - freq_dist = dist(wspd, var_to_bin_against=None, bins=np.arange(-0.5, max_speed+1, 1), bin_labels=None, + freq_dist = dist(wspd, var_to_bin_against=None, bins=np.arange(-0.5, max_speed + 1, 1), bin_labels=None, x_label='Wind Speed [m/s]', max_y_value=max_y_value, aggregation_method='%frequency', return_data=True) if return_data: @@ -823,11 +823,11 @@ def _get_direction_binned_series(sectors, direction_series, direction_bin_array= direction_bin_array = utils.get_direction_bin_array(sectors) zero_centered = True else: - sectors = len(direction_bin_array)-1 + sectors = len(direction_bin_array) - 1 zero_centered = False if direction_bin_labels is None: direction_bin_labels = _get_direction_bin_labels(sectors, direction_bin_array, zero_centered) - direction_binned_series = _binned_direction_series(direction_series, sectors, direction_bin_array)\ + direction_binned_series = _binned_direction_series(direction_series, sectors, direction_bin_array) \ .rename('direction_bin') return direction_binned_series, direction_bin_labels, sectors, direction_bin_array, zero_centered @@ -885,11 +885,12 @@ def dist_by_dir_sector(var_series, direction_series, sectors=12, aggregation_met _get_direction_binned_series(sectors, direction_series, direction_bin_array, direction_bin_labels) data = pd.concat([var_series.rename('data'), direction_binned_series], join='inner', axis=1) if aggregation_method == '%frequency': - result = data.groupby(['direction_bin'], observed=False)['data'].count().rename('%frequency')/len(data) * 100.0 + result = data.groupby(['direction_bin'], observed=False)['data'].count().rename('%frequency') / len( + data) * 100.0 else: result = data.groupby(['direction_bin'], observed=False)['data'].agg(aggregation_method) - for i in range(1, sectors+1): + for i in range(1, sectors + 1): if not (i in result.index): result[i] = 0.0 result = result.sort_index() @@ -926,7 +927,7 @@ def _get_dist_matrix_by_dir_sector(var_series, var_to_bin_series, direction_seri if aggregation_method == '%frequency': counts = data.groupby([var_to_bin_series.name, 'direction_bin'], observed=False).count().unstack(level=-1) - distribution = counts/(counts.sum().sum()) * 100.0 + distribution = counts / (counts.sum().sum()) * 100.0 else: distribution = data.groupby([var_to_bin_series.name, 'direction_bin'], observed=False).agg(aggregation_method).unstack(level=-1) @@ -1359,8 +1360,8 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 graph = bw_plt.plot_rose_with_gradient(result, plot_bins=plot_bins, plot_labels=plot_labels, percent_symbol=freq_as_percentage) if text_msg_out: - word_list = textwrap.TextWrapper(width=graph.get_size_inches()[0]*10).wrap(text=text_msg_out) - graph.text(.5, 10**-len(word_list), "\n ".join(map(str, word_list)), ha='center', fontsize=14) + word_list = textwrap.TextWrapper(width=graph.get_size_inches()[0] * 10).wrap(text=text_msg_out) + graph.text(.5, 10 ** -len(word_list), "\n ".join(map(str, word_list)), ha='center', fontsize=14) if direction_bin_labels is not None: result.columns = direction_bin_labels @@ -1443,7 +1444,7 @@ def time_continuity_gaps(data: pd.DataFrame, minimum_gap_length: Optional[pd.Tim continuity['Days Lost'] = continuity['Time gap'] / pd.Timedelta('1 days') # Remove indexes where no days are lost before returning - + if resolution.kwds == {'months': 1}: index_filter = ~continuity['Days Lost'].isin([28, 29, 30, 31]) elif resolution.kwds == {'years': 1}: @@ -1470,7 +1471,7 @@ def time_continuity_gaps(data: pd.DataFrame, minimum_gap_length: Optional[pd.Tim return filtered - + def coverage(data, period='1M', aggregation_method='mean', data_resolution=None): """ Get the data coverage over the period specified. @@ -1607,7 +1608,7 @@ def custom_agg(x): if not isinstance(aggregation_method, str): aggregation_method = aggregation_method.__name__ if return_data: - return bw_plt.plot_12x24_contours(pvt_tbl, label=(var_name_label, aggregation_method)),\ + return bw_plt.plot_12x24_contours(pvt_tbl, label=(var_name_label, aggregation_method)), \ pvt_tbl return bw_plt.plot_12x24_contours(pvt_tbl, label=(var_name_label, aggregation_method)) @@ -1731,7 +1732,7 @@ class give as input a pandas.DataFrame having first column name as 'windspeed' a var_to_bin_against=ti['wspd'], bins=speed_bin_array, bin_labels=None, - aggregation_method=lambda x: x.quantile(percentile/100), + aggregation_method=lambda x: x.quantile(percentile / 100), return_data=True)[-1].rename("Rep_TI"), dist(var_to_bin=ti['Turbulence_Intensity'], var_to_bin_against=ti['wspd'], @@ -1861,7 +1862,6 @@ def twelve_by_24(wspd, wspd_std, return_data=False, var_name_label='Turbulence I def _calc_ratio(var_1, var_2, min_var=3, max_var=50): - var_1_bounded = var_1[(var_1 >= min_var) & (var_1 < max_var)] var_2_bounded = var_2[(var_2 >= min_var) & (var_2 < max_var)] ratio = pd.concat([var_1_bounded.rename('var_1'), var_2_bounded.rename('var_2')], axis=1, join='inner') @@ -2030,7 +2030,8 @@ def sector_ratio(wspd_1, wspd_2, wdir, sectors=72, min_wspd=3, direction_bin_arr sec_rat_dist = sec_rat_dist.rename('Mean_Sector_Ratio').to_frame() sec_rats_dists[sensor_pair] = sec_rat_dist - fig = bw_plt.plot_sector_ratio(sec_ratio=sec_rats, wdir=wdir_dict, sec_ratio_dist=sec_rats_dists, col_names=col_names, + fig = bw_plt.plot_sector_ratio(sec_ratio=sec_rats, wdir=wdir_dict, sec_ratio_dist=sec_rats_dists, + col_names=col_names, boom_dir_1=boom_dir_1, boom_dir_2=boom_dir_2, radial_limits=radial_limits, annotate=annotate, figure_size=figure_size) @@ -2048,57 +2049,113 @@ def calc_air_density(temperature: Union[float, pd.Series], elevation_site: Union[float, int] = None, lapse_rate: float = AIR_DENSITY_LAPSE_RATE, specific_gas_constant: float = 286.9, - rel_humidity_percent: Union[float, pd.Series] = None + rel_humidity_percent: Union[float, pd.Series] = None, + dew_point_temperature_degC: Union[float, pd.Series] = None, + calc_method: Optional[str] = 'IEC' ) -> Union[float, pd.Series]: """ - Calculates air density for a given air temperature (temperature), air pressure (pressure) - and relative humidity (rel_humidity_percent) using method outlined in IEC Standard (61400-12-1). - + Calculates air density for a given air temperature (temperature), air pressure (pressure) + and either relative humidity (rel_humidity_percent) or dew point temperature (dew_point_temperature_degC). + + **Calculation method:** + The air density is calculated as the sum of the densities of the dry air component + and the water vapour component using the equation: + + air_density = (P_d / (R_d * T)) + (P_v / (R_v * T)) + + where: + P_v = water vapour pressure [Pa]. + There are several methods to calculate water vapour pressure - use the calc_method argument to select + one of the following methods from bw.analyse.analyse._calc_water_vapour_pressure(): + - 'IEC' (default): Requires air temperature and relative humidity. + - 'HermanWobus_from_rel_humidity': Requires air temperature and relative humidity. + - 'HermanWobus_from_dew_point': Requires dew point temperature. + Note that regardless of the calc_method for water vapour pressure, + air temperature and air pressure are required inputs to calculate air density. + P_d = pressure of dry air [Pa] = total air pressure - water vapour pressure (P_v). + R_d = specific gas constant for dry air (287.05 J/kg*K). + See Note on specific_gas_constant argument for 'IEC' method below. + R_v = specific gas constant for water vapour (461.495 J/kg*K). + T = air temperature in Kelvin (°C + 273.15). + + For more detailed explanation of the calculation methods, see the following: + - 'IEC' Method Description: + IEC 61400-12-1 (2017) Wind power generation systems - Part 12-1: Power performance + measurement of electricity producing wind turbines (p. 41) + - Herman Wobus approximation for water vapour pressure and relating + air temperature, dew point temperature and relative humidity: + https://wahiduddin.net/calc/density_altitude.htm + https://help.emd.dk/knowledgebase/content/ReferenceManual/DensityOfAir.pdf + + Note on specific_gas_constant argument for 'IEC' method: Note that the value of the specific_gas_constant argument is used only when rel_humidity_percent is None. - This specific_gas_constant argument and its default value of 286.9 are retained to maintain backwards - compatibility for when rel_humidity_percent is None. + This specific_gas_constant argument and its default value of 286.9 are + retained to maintain backwards compatibility for when rel_humidity_percent is None. However, if rel_humidity_percent is not None, the specific_gas_constant is ignored as the air density - calculation which incorporates humidity depends on the two specific gas constants for dry air - (287.05 J/kg*K) and water vapour (461.5 J/kg*K) and therefore the specific_gas_constant argument is redundant. + calculation which incorporates relative humidity depends on the two specific gas constants for dry air + (287.05 J/kg*K) and water vapour (461.495 J/kg*K) and therefore the specific_gas_constant argument is redundant. WARNING: The `specific_gas_constant` argument will be removed in a future 3.0 release of brightwind. - Function gives option to scale air density to the elevation of the site if both the reference elevation - (elevation_ref) and site elevation (elevation_site) are provided. This scaling assumes that air density - decreases with increasing altitude above the earth's surface - a default air density lapse rate of -0.113 kg/m3/km + Function gives option to scale air density to the elevation of the site if both the reference elevation + (elevation_ref) and site elevation (elevation_site) are provided. This scaling assumes that air density + decreases with increasing altitude above the earth's surface - a default air density lapse rate of -0.113 kg/m3/km is adopted. Taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) - WARNING: Option of scaling air density to height will be removed in a future 3.0 release of brightwind. Please call + WARNING: Option of scaling air density to height will be removed in a future 3.0 release of brightwind. Please call `scale_air_density_to_height()` separately instead. - :param temperature: Air temperature values in degree Celsius - :type temperature: float or pandas.Series - :param pressure: Air pressure values in hectopascal, hPa or millibar, mbar (Note 1hPa = 1mbar) - (1013.25 hPa = 101,325 Pa = 101.325 kPa = 1 atm = 1013.25 mbar) - :type pressure: float or pandas.Series - :param elevation_ref: Elevation, in meters, of the reference air density location (and air temperature, - air pressure, relative humidity measurements used for the calculation). - :type elevation_ref: float or int - :param elevation_site: Elevation, in meters, of the site location for which air density is calculated. - Scaled air density value(s) are calculated from elevation_ref to elevation_site - if both are specified. - :type elevation_site: float or int - :param lapse_rate: Air density lapse rate kg/m^3/km. Default is -0.113 kg/m^3/km. - :type lapse_rate: float - :param specific_gas_constant: Specific gas constant R, for moist air, in J/(kg*K). Default is 286.9 J/(kg*K). - If rel_humidity_percent is not None, this argument is ignored and the specific - gas constant for dry air of 287.05 J/(kg*K) is used instead. - If rel_humidity_percent is None, then provided the user is unconcerned with - backwards compatibility, the user should set the specific_gas_constant to 287.05 or - set rel_humidity_percent to 0 to use the specific gas constants for dry air. - :type specific_gas_constant: float - :param rel_humidity_percent: Relative humidity values as a percentage. Default is None. If None, the air density - calculation ignores humidity. - :type rel_humidity_percent: float or pandas.Series - :return: Air density in kg/m^3. Output type depends on type(temperature), type(pressure), - type(rel_humidity_percent) provided. If all inputs are float, output is float. - If any input is pandas.Series, output is pandas.Series. - :rtype: float or pandas.Series + :param temperature: Air temperature values in degrees Celsius + :type temperature: float or pandas.Series + :param pressure: Air pressure values in hectopascal, hPa or millibar, mbar (Note 1hPa = 1mbar) + (1013.25 hPa = 101,325 Pa = 101.325 kPa = 1 atm = 1013.25 mbar) + :type pressure: float or pandas.Series + :param elevation_ref: Elevation, in meters, of the reference air density location + (and air temperature, air pressure, relative humidity measurements used for + the calculation). + :type elevation_ref: float or int + :param elevation_site: Elevation, in meters, of the site location for which air density is calculated. + Scaled air density value(s) are calculated from elevation_ref to elevation_site + if both are specified. + :type elevation_site: float or int + :param lapse_rate: Air density lapse rate kg/m^3/km. Default is -0.113 kg/m^3/km. + :type lapse_rate: float + :param specific_gas_constant: Specific gas constant R, for moist air, in J/(kg*K). Default is 286.9 J/(kg*K). + When calc_method = 'IEC': + If rel_humidity_percent is not None, this argument is ignored and the specific + gas constant for dry air of 287.05 J/(kg*K) is used instead. + If rel_humidity_percent is None, then provided the user is unconcerned with + backwards compatibility, the user should set the specific_gas_constant to + 287.05 or set rel_humidity_percent to 0 to use the specific gas constants + for dry air. For other calc_method options, this argument is ignored. + :type specific_gas_constant: float + :param rel_humidity_percent: Relative humidity values as a percentage. Default is None. + If None and calc_method is 'IEC', + the air density calculation assumes a relative humidity of 0% (dry air). + :type rel_humidity_percent: float or pandas.Series + :param dew_point_temperature_degC: Dew point temperature values in degrees Celsius. Default is None. + :type dew_point_temperature_degC: float or pandas.Series + :param calc_method: Method to calculate water vapour pressure using + bw.analyse.analyse._calc_water_vapour_pressure(). + Options are: + - 'IEC' (default): + Inputs are air temperature (temperature) and + relative humidity (rel_humidity_percent). + If rel_humidity_percent input to bw.calc_air_density() is None, + a value of 0% is assumed for the water vapour pressure calculation. + - 'HermanWobus_from_rel_humidity': + Requires air temperature (temperature) and + relative humidity (rel_humidity_percent). + - 'HermanWobus_from_dew_point': + Requires dew point temperature (dew_point_temperature_degC). + Note that regardless of the calc_method for water vapour pressure, + air temperature and air pressure are required inputs to calculate air density. + :type calc_method: str + :return: Air density in kg/m^3. Output type depends on type(temperature), type(pressure), + type(rel_humidity_percent) or type(dew_point_temperature_degC) provided. + If all inputs are float, output is float. + If any input is pandas.Series, output is pandas.Series. + :rtype: float or pandas.Series **Example usage** :: @@ -2106,6 +2163,7 @@ def calc_air_density(temperature: Union[float, pd.Series], data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) + # IEC Method Examples (Default): # Calculate air density from input air temperature and air pressure series # (rel_humidity_percent=None and specific_gas_constant=286.9 by default) bw.calc_air_density(data.T2m, data.P2m).head(5) @@ -2159,31 +2217,58 @@ def calc_air_density(temperature: Union[float, pd.Series], bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200) # 1.198 + # HermanWobus_from_rel_humidity Method Example: + # Calculate air density from input air temperature and air pressure series + # (rel_humidity_percent=None and specific_gas_constant=286.9 by default) + bw.calc_air_density(data.T2m, data.P2m, rel_humidity_percent=data.RH2m, + calc_method='HermanWobus_from_rel_humidity').head(5) + # Timestamp + # 2016-01-09 15:30:00 1.186297 + # 2016-01-09 15:40:00 1.186666 + # 2016-01-09 17:00:00 1.183138 + # 2016-01-09 17:10:00 1.183919 + # 2016-01-09 17:20:00 1.184333 + # dtype: float64 + + # HermanWobus_from_dew_point Method Example: + # Calculate air density from input air temperature, air pressure and dew point temperature series + + dew_point_temp = data.T2m - 2 # Example dew point temperature series + bw.calc_air_density(data.T2m, data.P2m, + dew_point_temperature_degC=dew_point_temp, + calc_method='HermanWobus_from_dew_point').head(5) + # Timestamp + # 2016-01-09 15:30:00 1.186716 + # 2016-01-09 15:40:00 1.187083 + # 2016-01-09 17:00:00 1.183568 + # 2016-01-09 17:10:00 1.184345 + # 2016-01-09 17:20:00 1.184756 + # dtype: float64 """ - # Raise errors if any inputs are DataFrames - if isinstance(temperature, pd.DataFrame): - raise ValueError("temperature cannot be provided as a DataFrame. Provide as float or pandas Series.") - if isinstance(pressure, pd.DataFrame): - raise ValueError("pressure cannot be provided as a DataFrame. Provide as float or pandas Series.") - if isinstance(rel_humidity_percent, pd.DataFrame): - raise ValueError("rel_humidity_percent cannot be provided as a DataFrame. Provide as float or pandas Series.") - - # Check dimensions of temperature, pressure and rel_humidity_percent if not float or int - if isinstance(temperature, pd.Series) and (isinstance(pressure, pd.Series)): - if len(temperature) != len(pressure): - raise ValueError("temperature and pressure must have the same dimensions.") - if isinstance(rel_humidity_percent, pd.Series): - if len(temperature) != len(rel_humidity_percent): - raise ValueError("temperature, pressure and rel_humidity_percent must have the same dimensions.") - - lapse_rate_per_m = lapse_rate * 0.001 # convert lapse rate from kg/m3/km to kg/m3/m - temp_K = temperature + 273.15 # to convert deg C to Kelvin. + # Validate required input types + _assert_function_variable_type(temperature, (float, int, pd.Series), 'temperature') + _assert_function_variable_type(pressure, (float, int, pd.Series), 'pressure') + _assert_series_index_match(temperature, pressure, 'temperature', 'pressure') + # Check type and dimensions if not float or int + if calc_method in ['IEC', 'HermanWobus_from_rel_humidity']: + if rel_humidity_percent is not None: + _assert_function_variable_type(rel_humidity_percent, (float, int, pd.Series), 'rel_humidity_percent') + _assert_series_index_match(temperature, rel_humidity_percent, + 'temperature, pressure', 'rel_humidity_percent') + elif calc_method == 'HermanWobus_from_dew_point': + if dew_point_temperature_degC is not None: + _assert_function_variable_type(dew_point_temperature_degC, (float, int, pd.Series), + 'dew_point_temperature_degC') + _assert_series_index_match(temperature, dew_point_temperature_degC, + 'temperature, pressure', 'dew_point_temperature_degC') + + lapse_rate_per_m = lapse_rate * 0.001 # convert lapse rate from kg/m3/km to kg/m3/m + temp_K = temperature + DEGREES_CELSIUS_TO_KELVIN # to convert deg C to Kelvin. press_Pa = pressure * 100 # to convert hPa to Pa - vapour_press = 0.0000205 * np.exp(0.0631846 * temp_K) - specific_gas_constant_water = 461.5 specific_gas_constant_dry_air = GAS_CONST_DRY_AIR + specific_gas_constant_water = GAS_CONST_WATER - if rel_humidity_percent is None: + if rel_humidity_percent is None and calc_method == 'IEC': # If relative humidity is None, the old method ignoring relative humidity is used. # To avoid a breaking change, the value of the specific_gas_constant argument is used. # (Default specific_gas_constant is 286.9 which is for moist air) @@ -2191,7 +2276,7 @@ def calc_air_density(temperature: Union[float, pd.Series], # the specific gas constant for dry air (287.05) which makes more physical sense given relative humidity is # taken as zero in the calculation below. gas_constant_air = specific_gas_constant - rel_hum_decimal = 0.0 + rel_humidity_percent = 0.0 # Deprecation warning warnings.warn( ( @@ -2199,52 +2284,411 @@ def calc_air_density(temperature: Union[float, pd.Series], "release of brightwind.\nNote that this value is used only when `rel_humidity_percent` is None." "\nPlease set `rel_humidity_percent` to 0 instead if you want to use the specific gas constant " "for dry air (287.05 J/kg*K)." - ), + ), DeprecationWarning, stacklevel=2 ) else: # Otherwise, if the rel_humidity_percent is not None (including = 0), the new method including relative humidity - # is used. The specific_gas_constant_dry_air of 287.05 is used as the air density calculation incorporating - # relative humidity is clearly split into a dry component and a moist component. Therefore it makes physical + # is used. The specific_gas_constant_dry_air of 287.05 is used as the air density calculation incorporating + # relative humidity is clearly split into a dry component and a moist component. Therefore it makes physical # sense to use the specific gas constant for dry air in this context. gas_constant_air = specific_gas_constant_dry_air - rel_hum_decimal = rel_humidity_percent * 0.01 - - # Contribution from dry air - dry_air_term = press_Pa / gas_constant_air - - # Contribution from water vapour - water_vapour_term = rel_hum_decimal * vapour_press * ( - (1 / gas_constant_air) - (1 / specific_gas_constant_water)) - - # Combine to get air density - air_density = (1 / temp_K) * (dry_air_term - water_vapour_term) - + + # If calc_method requires dew point temperature, ensure it is always <= temperature: + if calc_method == 'HermanWobus_from_dew_point': + if dew_point_temperature_degC is None: + raise ValueError("For 'HermanWobus_from_dew_point' calc_method, dew_point_temperature_degC must be" + " provided.") + # Raise error if dew point temperature is greater than air temperature + comparison = dew_point_temperature_degC > temperature + msg = "dew_point_temperature_degC cannot be greater than temperature." + # For pandas Series, ignore NaNs when checking + if isinstance(comparison, pd.Series): + if comparison.fillna(False).any(): + raise ValueError(msg) + # For scalar booleans + else: + if comparison: + raise ValueError(msg) + + # Calculate partial pressure of water vapour + water_vapour_press_Pa = _calc_water_vapour_pressure_Pa(air_temperature_degC=temperature, + rel_humidity_percent=rel_humidity_percent, + dew_point_temperature_degC=dew_point_temperature_degC, + calc_method=calc_method) + # Calculate partial pressure of dry air + dry_air_press_Pa = press_Pa - water_vapour_press_Pa + + # Calculate air density + air_density = dry_air_press_Pa / (gas_constant_air * temp_K) + water_vapour_press_Pa / ( + specific_gas_constant_water * temp_K) + + # Scale air density to site elevation if both elevation_ref and elevation_site are provided if elevation_ref is not None and elevation_site is not None: # Deprecation warning warnings.warn( ( "\nScaling air density to height within `calc_air_density()` is deprecated and will be removed in a " "future 3.0 release of brightwind.\nPlease call `scale_air_density_to_height()` separately instead. \n" - ), + ), DeprecationWarning, stacklevel=2 ) # Perform extrapolation scaled_air_density = scale_air_density_to_height(air_density, - elevation_ref, - elevation_site, - lapse_rate_per_m) + elevation_ref, + elevation_site, + lapse_rate_per_m) return round(scaled_air_density, 3) - + # Validate elevation arguments if elevation_site is not None and elevation_ref is None: raise TypeError("Specify value of elevation_ref (float or int) when elevation_site is provided.") elif elevation_ref is not None and elevation_site is None: raise TypeError("Specify value of elevation_site (float or int) when elevation_ref is provided.") - + return air_density +def _calc_water_saturation_vapour_pressure_Pa(air_temperature_degC: Union[float, pd.Series] + ) -> Union[float, pd.Series]: + """ + Calculate the saturation vapour pressure of water vapour for a given air temperature using + Herman Wobus polynomial approximation. + + The saturation vapour pressure represents the maximum partial pressure of water vapour + that can exist in the air at a specified temperature before the air becomes saturated + and condensation occurs. + + For more detailed references see the following: + https://wahiduddin.net/calc/density_altitude.htm + https://help.emd.dk/knowledgebase/content/ReferenceManual/DensityOfAir.pdf + + **Calculation method:** + The function applies a polynomial approximation to compute the saturation vapour pressure + of water vapour over liquid water based on temperature (T), following: + E_s(T) = e_s0 / (p(T))^8 + where: + e_s0 = 6.1078 hPa + p(T) = c₀ + T(c₁ + T(c₂ + T(c₃ + T(c₄ + T(c₅ + T(c₆ + T(c₇ + T(c₈ + T·c₉)))))))) + and coefficients: + c₀ = 0.99999683 + c₁ = -0.90826951x10⁻² + c₂ = 0.78736169x10⁻⁴ + c₃ = -0.61117958x10⁻⁶ + c₄ = 0.43884187x10⁻⁸ + c₅ = -0.29883885x10⁻¹⁰ + c₆ = 0.21874425x10⁻¹² + c₇ = -0.17892321x10⁻¹⁴ + c₈ = 0.11112018x10⁻¹⁶ + c₉ = -0.30994571x10⁻¹⁹ + The output saturation vapour pressure is converted from hPa to Pa by multiplying by 100. + + :param air_temperature_degC: Air temperature in degrees Celsius. + :type air_temperature_degC: float or pandas.Series + :return: Saturation vapour pressure in pascals (Pa). + Output type depends on input type. If input is float, + output is float. If input is pandas.Series, + output is pandas.Series. + :rtype: float or pandas.Series + + import brightwind as bw + + # Calculate saturation vapour pressure for 20°C air temperature + bw.analyse.analyse._calc_water_saturation_vapour_pressure_Pa(20.0) + # 2337.237477998109 Pa + + # Calculation saturation vapour pressure for a series of air temperature values + data = bw.load_csv(bw.demo_datasets.demo_data) + data = bw.apply_cleaning(data, bw.demo_datasets.demo_cleaning_file) + bw.analyse.analyse._calc_water_saturation_vapour_pressure_Pa(data.T2m.tail(3)) + # Timestamp + # 2017-11-23 10:30:00 647.322512 + # 2017-11-23 10:40:00 651.117108 + # 2017-11-23 10:50:00 647.322512 + # Name: T2m, dtype: float64 + """ + e_s0 = 6.1078 + c0 = 0.99999683 + c1 = -0.90826951e-2 + c2 = 0.78736169e-4 + c3 = -0.61117958e-6 + c4 = 0.43884187e-8 + c5 = -0.29883885e-10 + c6 = 0.21874425e-12 + c7 = -0.17892321e-14 + c8 = 0.11112018e-16 + c9 = -0.30994571e-19 + p = (c0 + air_temperature_degC * + (c1 + air_temperature_degC * + (c2 + air_temperature_degC * + (c3 + air_temperature_degC * + (c4 + air_temperature_degC * + (c5 + air_temperature_degC * + (c6 + air_temperature_degC * + (c7 + air_temperature_degC * + (c8 + air_temperature_degC * c9))))))))) + E_s_hPa = e_s0 / (p ** 8) + # Convert from hPa to Pa + E_s_Pa = E_s_hPa * 100 + return E_s_Pa + + +def _calc_water_vapour_pressure_Pa(air_temperature_degC: Optional[Union[float, pd.Series]] = None, + rel_humidity_percent: Optional[Union[float, pd.Series]] = None, + dew_point_temperature_degC: Optional[Union[float, pd.Series]] = None, + calc_method: Optional[str] = 'IEC' + ) -> Union[float, pd.Series]: + """ + Calculate the water vapour pressure in Pascals (Pa) using one of the following methods: + 1. 'IEC': + - Requires air temperature (in degrees Celsius) and relative humidity (as a percentage). + + **Calculation method:** + The vapour pressure (P_v) is calculated using the formula provided in IEC Standard (61400-12-1): + P_v = (RH / 100) x 0.0000205 x exp(0.0631846 x T_K) + where: + P_v = actual water vapour pressure (Pa) + RH = relative humidity (%) + T_K = air temperature in Kelvin (K) = T(°C) + 273.15 + + 2. 'HermanWobus_from_rel_humidity': + - Requires air temperature (in degrees Celsius) and relative humidity (as a percentage). + + **Calculation method:** + Relative humidity is defined as the ratio (%) of the actual vapour pressure to the saturation + vapour pressure at a given temperature. To find the actual vapour pressure (P_v), the + saturation vapour pressure at the given air temperature is multiplied by the relative humidity: + P_v = (RH / 100) x E_s(T) + where: + P_v = actual water vapour pressure (Pa). + RH = relative humidity (%). + E_s(T) = saturation vapour pressure (Pa) at air temperature T (°C), + computed using the Herman Wobus Approximation - for details + see `_calc_water_saturation_vapour_pressure_Pa()`. + + 3. 'HermanWobus_from_dew_point': + - Requires dew point temperature (in degrees Celsius). + + **Calculation method:** + The dew point temperature is the temperature at which the air becomes saturated when cooled at constant + pressure without changing the moisture content. Therefore, the vapour pressure (P_v) of the air at its + actual temperature is equal to the saturation vapour pressure at the dew point temperature: + P_v = E_s(T_d) + where: + P_v = actual water vapour pressure (Pa). + T_d = dew point temperature (°C). + E_s(T_d) = saturation vapour pressure (Pa) at the dew point temperature (°C) + computed using the Herman Wobus Approximation - for details + see `_calc_water_saturation_vapour_pressure_Pa()`. + + :param air_temperature_degC: Air temperature in degrees Celsius. + :type air_temperature_degC: float, pd.Series + :param rel_humidity_percent: Relative humidity as a percentage. + :type rel_humidity_percent: float, pd.Series + :param dew_point_temperature_degC: Dew point temperature in degrees Celsius. + :type dew_point_temperature_degC: float, pd.Series + :param calc_method: Method to use for calculation. Options are: + - 'IEC' (default): Uses air temperature and relative humidity. + - 'HermanWobus_from_rel_humidity': Uses air temperature and relative humidity. + - 'HermanWobus_from_dew_point': Uses dew point temperature. + :type calc_method: str + :return: Water vapour pressure in Pascals (Pa). + Output type depends on input type. + If input all inputs are float, output is float. + If any input is pandas.Series, output is pandas.Series. + :rtype: float, pd.Series + + **Example usage** + :: + import brightwind as bw + + data = bw.load_csv(bw.demo_datasets.demo_data) + data = bw.apply_cleaning(data, bw.demo_datasets.demo_cleaning_file) + + # Examples Method 1: IEC (default) + # Calculate water vapour pressure for 60% relative humidity at 20°C using HermanWobus method + bw.analyse.analyse._calc_water_vapour_pressure_Pa(air_temperature_degC=20.0, + rel_humidity_percent=60.0) + # 1361.9246939756576 + + # Calculate water vapour pressure from a pandas Series of air temperature and relative humidity data + bw.analyse.analyse._calc_water_vapour_pressure_Pa(air_temperature_degC=data['T2m'].tail(5), + rel_humidity_percent=data['RH2m'].tail(5)) + + # Timestamp + # 2017-11-23 10:10:00 678.827053 + # 2017-11-23 10:20:00 664.962799 + # 2017-11-23 10:30:00 674.830715 + # 2017-11-23 10:40:00 678.293320 + # 2017-11-23 10:50:00 674.830715 + # dtype: float64 + + # Examples Method 2: HermanWobus_from_rel_humidity + # Calculate vapour pressure for 60% relative humidity at 20°C using HermanWobus method + bw.analyse.analyse._calc_water_vapour_pressure_Pa(air_temperature_degC=20.0, + rel_humidity_percent=60.0, + calc_method='HermanWobus_from_rel_humidity') + # 1402.3424867988654 + + # Calculate water vapour pressure from a pandas Series of temperature and relative humidity data + bw.analyse.analyse._calc_water_vapour_pressure_Pa(air_temperature_degC=data['T2m'].tail(5), + rel_humidity_percent=data['RH2m'].tail(5), + calc_method='HermanWobus_from_rel_humidity') + + # Timestamp + # 2017-11-23 10:10:00 651.978790 + # 2017-11-23 10:20:00 637.799429 + # 2017-11-23 10:30:00 647.322512 + # 2017-11-23 10:40:00 651.117108 + # 2017-11-23 10:50:00 647.322512 + # dtype: float64 + + # Examples Method 3: HermanWobus_from_dew_point + # Calculate vapour pressure for a dew point temperature of 10°C + bw.analyse.analyse._calc_water_vapour_pressure_Pa(dew_point_temperature_degC=10, + calc_method='HermanWobus_from_dew_point') + # 1227.2296498322416 + + # Calculate water vapour pressure from a pandas Series of dew point temperatures + import pandas as pd + + dew_pt = pd.Series([10, 12, 9]) + bw.analyse.analyse._calc_water_vapour_pressure_Pa(dew_point_temperature_degC=dew_pt, + calc_method='HermanWobus_from_dew_point') + + # 0 1227.229650 + # 1 1401.709287 + # 2 1147.394524 + # dtype: float64 + """ + if calc_method not in ['IEC', 'HermanWobus_from_rel_humidity', 'HermanWobus_from_dew_point']: + raise ValueError("Invalid calc_method. Choose from 'IEC', 'HermanWobus_from_rel_humidity'," + "'HermanWobus_from_dew_point'.") + + if calc_method == 'IEC': + + # Validate required inputs + if air_temperature_degC is None or rel_humidity_percent is None: + raise ValueError("For 'IEC' calc_method, both air_temperature_degC and rel_humidity_percent must be" + " provided.") + _assert_function_variable_type(air_temperature_degC, (float, int, pd.Series), 'air_temperature_degC') + _assert_function_variable_type(rel_humidity_percent, (float, int, pd.Series), 'rel_humidity_percent') + # If inputs are Series, check they have same index + _assert_series_index_match(air_temperature_degC, rel_humidity_percent, + 'air_temperature_degC', 'rel_humidity_percent') + + if dew_point_temperature_degC is not None: + warnings.warn("dew_point_temperature_degC input not required in water vapour pressure calculation" + " as calc_method is 'IEC'.", + UserWarning, stacklevel=2) + + rel_humidity_decimal = 0.01 * rel_humidity_percent # convert percent to decimal. + air_temperature_K = air_temperature_degC + DEGREES_CELSIUS_TO_KELVIN # convert deg C to Kelvin. + + # Calculate water vapour pressure using IEC formula + water_vapour_press_Pa = rel_humidity_decimal * 0.0000205 * np.exp(0.0631846 * air_temperature_K) + + if calc_method == 'HermanWobus_from_rel_humidity': + + # Validate required inputs + if air_temperature_degC is None or rel_humidity_percent is None: + raise ValueError("For 'HermanWobus_from_rel_humidity' calc_method, both air_temperature_degC" + " and rel_humidity_percent must be provided.") + _assert_function_variable_type(air_temperature_degC, (float, int, pd.Series), 'air_temperature_degC') + _assert_function_variable_type(rel_humidity_percent, (float, int, pd.Series), 'rel_humidity_percent') + # If inputs are Series, check they have same index + _assert_series_index_match(air_temperature_degC, rel_humidity_percent, + 'air_temperature_degC', 'rel_humidity_percent') + + if dew_point_temperature_degC is not None: + warnings.warn("dew_point_temperature_degC input not required in water vapour pressure calculation" + " as calc_method is 'HermanWobus_from_rel_humidity'.", + UserWarning, stacklevel=2) + + rel_humidity_decimal = 0.01 * rel_humidity_percent # convert percent to decimal. + + # Calculate water vapour pressure from air temperature and relative humidity using Herman Wobus approximation + water_vapour_press_Pa = (rel_humidity_decimal * + _calc_water_saturation_vapour_pressure_Pa(air_temperature_degC)) + + if calc_method == 'HermanWobus_from_dew_point': + + # Validate required inputs + if dew_point_temperature_degC is None: + raise ValueError("For 'HermanWobus_from_dew_point' calc_method, dew_point_temperature_degC must be" + " provided.") + _assert_function_variable_type( + dew_point_temperature_degC, (float, int, pd.Series), 'dew_point_temperature_degC' + ) + + if air_temperature_degC is not None or rel_humidity_percent is not None: + warnings.warn("Water vapour pressure calculated based on dew_point_temperature_degC as calc_method is" + " 'HermanWobus_from_dew_point'.\n Air temperature and relative humidity inputs " + " not required in water vapour pressure calculation.", + UserWarning, stacklevel=2) + + # Calculate water vapour pressure from dew point temperature using Herman Wobus approximation + water_vapour_press_Pa = _calc_water_saturation_vapour_pressure_Pa(dew_point_temperature_degC) + + return water_vapour_press_Pa + + +def _assert_function_variable_type(variable: Any, + expected_type: Union[type, tuple], + variable_name: str) -> None: + """ + Assert whether variable has expected type. If not, raise an error. + + :param variable: Variable for which type is checked. + :type variable: any + :param expected_type: Expected type(s) e.g. float or (float, int) + :type expected_type: type or tuple + :param variable_name: Name of the variable to be used in the error message. + :type variable_name: str + :returns: Raises an error if variable type is not as expected. + :rtype: None + + **Example usage** + :: + import brightwind as bw + a = [1, 2] + + bw.analyse.analyse._assert_function_variable_type(a, float, 'number') + # TypeError: 'number' must be type: , received: list + bw.analyse.analyse._assert_function_variable_type(a, (float, int), 'number') + # TypeError: 'number' must be type: (, ), received: list + """ + if not isinstance(variable, expected_type): + raise TypeError(f"'{variable_name}' must be type: {expected_type}, received: {type(variable).__name__}.") + + +def _assert_series_index_match(series1: pd.Series, + series2: pd.Series, + series1_name: str = None, + series2_name: str = None): + """ + Assert whether the index of series1 matches the index of series2. + + :param series1: First pandas series. + :type series1: pd.Series + :param series2: Second pandas series. + :type series2: pd.Series + :param series1_name: Name of first pandas series. + :param series2_name: Name of second pandas series. + :returns: Raises an error if index of series1 does not match index of series2. + :rtype: None + """ + if isinstance(series1, pd.Series) and isinstance(series2, pd.Series): + if series1_name is None: + series1_name = 'series1' + if series2_name is None: + series2_name = 'series2' + if len(series1) != len(series2): + raise ValueError(f"{series1_name} and {series2_name} must have the same dimensions.") + elif not (series1.index == series2.index).all(): + raise ValueError(f"{series1_name} and {series2_name} must have the same index.") + else: + return True diff --git a/brightwind/transform/scale.py b/brightwind/transform/scale.py index 108026c6..6cc0497e 100644 --- a/brightwind/transform/scale.py +++ b/brightwind/transform/scale.py @@ -1,6 +1,8 @@ from typing import Union import numpy as np import pandas as pd +from brightwind.utils.constants import (ACCEL_DUE_TO_GRAVITY, TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE, + GAS_CONST_DRY_AIR, AIR_DENSITY_LAPSE_RATE) __all__ = ['apply_scale_factor', 'linear_transform', @@ -8,19 +10,6 @@ 'scale_air_temperature_to_height', 'scale_air_pressure_to_height'] -# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere -ACCEL_DUE_TO_GRAVITY = 9.80665 - -# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere -TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 - -# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere -GAS_CONST_DRY_AIR = 287.05 - -# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) -AIR_DENSITY_LAPSE_RATE = -0.113 - - def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], scale_factor: Union[float, int] ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: diff --git a/brightwind/utils/constants.py b/brightwind/utils/constants.py new file mode 100644 index 00000000..4ea7812c --- /dev/null +++ b/brightwind/utils/constants.py @@ -0,0 +1,21 @@ +""" +This file contains constant numbers, strings, etc used elsewhere in the brightwind library +""" + +# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere +GAS_CONST_DRY_AIR = 287.05 + +# Specific gas constant for water vapour (J/K/kg or m2/K/s2) +GAS_CONST_WATER = 461.495 + +# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) +AIR_DENSITY_LAPSE_RATE = -0.113 + +# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere +ACCEL_DUE_TO_GRAVITY = 9.80665 + +# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere +TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + +# Offset to convert degrees Celsius to Kelvin +DEGREES_CELSIUS_TO_KELVIN = 273.15 diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index 2c56188a..b3911e5a 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -3,7 +3,6 @@ import os import json from jsonschema import Draft7Validator -from typing import Union __all__ = ['slice_data', 'validate_coverage_threshold', @@ -254,7 +253,3 @@ def validate_json(json_to_check, schema): print(f"Failed schema part: {error.get('schema_path')}\n") return data_is_valid - - - - diff --git a/tests/test_analyse.py b/tests/test_analyse.py index e61e1d3b..c71f8051 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -638,39 +638,47 @@ def test_ti_by_sector(): def test_calc_air_density(): - - # test Series inputs + + # Test error for invalid calc_method + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(DATA.T2m, DATA.P2m, calc_method='invalid_method') + assert str(except_info.value) == ("Invalid calc_method. Choose from 'IEC', 'HermanWobus_from_rel_humidity'," + "'HermanWobus_from_dew_point'.") + + # Tests for default calc_method = 'IEC' + + # Test Series inputs assert list(round(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5), 6).values - ) == [1.199177, 1.199838, 1.199794, 1.199439, 1.201066] + ) == [1.199177, 1.199838, 1.199794, 1.199439, 1.201066] assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5).values - pd.Series([1.19918, 1.19984, 1.19979, 1.19944, 1.20107])) < 1e-3).all() assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m, rel_humidity_percent=DATA.RH2m).tail(5).values - pd.Series([1.19529, 1.19601, 1.19592, 1.19555, 1.19719])) < 1e-3).all() assert list(bw.calc_air_density(DATA.T2m, DATA.P2m, specific_gas_constant=287.05).head(5).round(6).dropna() ) == [1.187064, 1.187458] - - #t test Series inputs with elevation adjustment + + # Test Series inputs with elevation adjustment assert list(bw.calc_air_density(DATA.T2m, DATA.P2m, elevation_ref=0, elevation_site=200).tail(5).values ) == [1.177, 1.177, 1.177, 1.177, 1.178] - # test float/int inputs + # Test float/int inputs assert bw.calc_air_density(15, 1013) == 1.2253503331640465 - assert (bw.calc_air_density(15, 1012, rel_humidity_percent=None, specific_gas_constant=287.05) == + assert (bw.calc_air_density(15, 1012, rel_humidity_percent=None, specific_gas_constant=287.05) == bw.calc_air_density(15, 1012, rel_humidity_percent=0)) assert round(bw.calc_air_density(15, 1012, rel_humidity_percent=0), 5) == 1.2235 assert round(bw.calc_air_density(15, 1012), 5) == 1.22414 assert abs(bw.calc_air_density(15, 1013, specific_gas_constant=287.05) - 1.22471) < 1e-3 assert abs(bw.calc_air_density(15, 1013, rel_humidity_percent=50) - 1.22093) < 1e-3 - assert round(bw.calc_air_density(15, 1013, rel_humidity_percent=50, + assert round(bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200), 5) == 1.198 - # test float/int inputs with elevation adjustment + # Test float/int inputs with elevation adjustment assert bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) == 1.203 assert bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200 ) - bw.transform.scale.scale_air_density_to_height( - bw.calc_air_density(15, 1013, rel_humidity_percent=50), 0, 200) <1e-3 - - # test errors + bw.calc_air_density(15, 1013, rel_humidity_percent=50), 0, 200) < 1e-3 + + # Test errors with pytest.raises(TypeError) as except_info: bw.calc_air_density(15, 1013, elevation_site=200) assert str(except_info.value) == "Specify value of elevation_ref (float or int) when elevation_site is provided." @@ -678,12 +686,61 @@ def test_calc_air_density(): bw.calc_air_density(15, 1013, elevation_ref=200) assert str(except_info.value) == "Specify value of elevation_site (float or int) when elevation_ref is provided." with pytest.raises(ValueError) as except_info: - bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) + bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) assert str(except_info.value) == "temperature and pressure must have the same dimensions." with pytest.raises(ValueError) as except_info: - bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) + bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) assert str(except_info.value) == "temperature, pressure and rel_humidity_percent must have the same dimensions." + # Tests for calc_method = 'HermanWobus_from_rel_humidity' + + # Check error is raised when rel_humidity_percent is not provided + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(0.711, 935, + calc_method='HermanWobus_from_rel_humidity') + assert str(except_info.value) == ("For 'HermanWobus_from_rel_humidity' calc_method, both air_temperature_degC" + " and rel_humidity_percent must be provided.") + + # Test Series inputs + assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m, rel_humidity_percent=DATA.RH2m, + calc_method='HermanWobus_from_rel_humidity').tail(5).values - + pd.Series([1.19541777, 1.19614486, 1.19605542, 1.19568365, 1.19732707]) < 1e-3)).all() + # Test float inputs + assert bw.calc_air_density(0.711, 935, rel_humidity_percent=50, + calc_method='HermanWobus_from_rel_humidity') - 1.1878427585014013 < 1e-6 + + # Tests for calc_method = 'HermanWobus_from_dew_point' + + # Check error is raised when dew_point_temperature_degC is not provided + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(0.711, 935, rel_humidity_percent=85, + calc_method='HermanWobus_from_dew_point') + assert str(except_info.value) == ("For 'HermanWobus_from_dew_point' calc_method, dew_point_temperature_degC must be" + " provided.") + + # Check error is raised when dew_point_temperature_degC > temperature + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(1, 935, dew_point_temperature_degC=5, + calc_method='HermanWobus_from_dew_point') + assert str(except_info.value) == ("dew_point_temperature_degC cannot be greater than temperature.") + + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(DATA.T2m.tail(3), DATA.P2m.tail(3), + dew_point_temperature_degC=DATA.T2m.tail(3) + 5, + calc_method='HermanWobus_from_dew_point') + assert str(except_info.value) == ("dew_point_temperature_degC cannot be greater than temperature.") + + # Test Series inputs + dew_point_temp = DATA.T2m - 0.5 + assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m, + dew_point_temperature_degC=dew_point_temp, + calc_method='HermanWobus_from_dew_point').tail(5).values - + pd.Series([1.19551981, 1.19621178, 1.19616594, 1.19579471, 1.19743759]) < 1e-3)).all() + # Test float inputs + assert bw.calc_air_density(0.711, 935, + dew_point_temperature_degC=-1.299, + calc_method='HermanWobus_from_dew_point') - 1.186717791022866 < 1e-6 + def test_dist_matrix_by_direction_sector(): bw.dist_matrix_by_dir_sector(var_series=DATA.Spd80mN, var_to_bin_by_series=DATA.Spd80mN, @@ -695,3 +752,12 @@ def test_dist_matrix_by_direction_sector(): bw.dist_matrix_by_dir_sector(DATA.Spd40mN, DATA.T2m, DATA.Dir38mS, var_to_bin_by_array=[-8, -5, 5, 10, 15, 20, 26], sectors=8) assert True + + +def test_calc_water_saturation_vapour_pressure_Pa(): + # test float input + assert abs(bw.analyse.analyse._calc_water_saturation_vapour_pressure_Pa(20) - 2337.237477998109) < 1e-4 + + # test series input + assert abs(bw.analyse.analyse._calc_water_saturation_vapour_pressure_Pa(DATA.T2m.tail(3)).values - + pd.Series([647.3225, 651.1171, 647.3225]) < 1e-4).all() diff --git a/tests/test_utils.py b/tests/test_utils.py index e48096b0..7a7f9614 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -26,3 +26,4 @@ def test_slice_data(): data_sliced = bw.utils.utils.slice_data(DATA, date_to='2017-10-23') assert data_sliced.index[0] == DATA.index[0] + \ No newline at end of file From 6624e0f78502597bb8a1eb444c2a2d7f0af2eef5 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Tue, 10 Feb 2026 12:43:06 +0000 Subject: [PATCH 29/35] [Iss581] Calculate relative humidity from dew point (#582) * iss #544 moved scaling functions to transform/scale.py * iss #539 added several functions for calculating air density, vapour pressure and saturation vapour pressure * iss #539 added tests * iss #539 updated changelog * iss #539 minor reformatting * iss #539 updated calc_air_density to handle three different calculation methods * iss #539 added examples, fixed formatting * iss #539 fixed formatting * iss #539 updated tests * iss #539 updated changelog * iss #539 moved constants to utils/constants.py * iss #539 fixed typo R_w * iss #539 updated to use global constant value for R_w * iss #539 added references to docstring of bw.calc_air_density * iss #581 add type check function to utils * iss #581 add calc_rel_humidity_from_dew_point * iss #581 added tests and examples * iss #539 tidied check dew point < temp * iss #539 updated assert type function name, docstring and added tests * iss #539 rewordings and format fixes * iss #581 merge fixes * iss #539 moved input type check from utils to be a hidden function in analyse.py * iss #539 update error msg wording * iss #539 add function to assert series indexes match * iss #539 added check for series index match * iss #539 formatting * iss #581 added check index matches * iss #581 updated changelog * iss #539 reduced checking code and automatic PEP8 fix pycharm * iss #581 changer errors to warnings * iss #581 changed error message * iss #581 changed else if to elif * iss #581 update warning message * iss #581 add reference --------- Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran Co-authored-by: BiancaMorandi <65398058+BiancaMorandi@users.noreply.github.com> --- CHANGELOG.md | 2 + brightwind/analyse/analyse.py | 89 ++++++++++++++++++++++++++++++++--- tests/test_analyse.py | 33 +++++++++++++ 3 files changed, 117 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65faf3fb..0ee66969 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.5.0] ### New Features and Enhancements 1. Updated `calc_air_density` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). +2. Added `calc_rel_humidity_from_dew_point` to calculate relative humidity from air temperature and dew point temperature. ([#581](https://github.com/brightwind-dev/brightwind/issues/581)). + ## [2.4.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index e9308e7c..0173ff8b 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -34,7 +34,8 @@ 'basic_stats', 'TI', 'sector_ratio', - 'calc_air_density'] + 'calc_air_density', + 'calc_rel_humidity_from_dew_point'] def dist_matrix(var_series, x_series, y_series, @@ -1404,7 +1405,7 @@ def time_continuity_gaps(data: pd.DataFrame, minimum_gap_length: Optional[pd.Tim :param minimum_gap_length: The minimum length time gap to report. Shorter time gaps will be filtered out of the returned DataFrame :type minimum_gap_length: Optional[pd.Timedelta] - :return: A table listing all the time gaps in the data that are not equal to the + :return: A table listing all the time gaps in the data that are not equal to the derived temporal resolution. :rtype: pd.DataFrame @@ -2303,15 +2304,15 @@ def calc_air_density(temperature: Union[float, pd.Series], " provided.") # Raise error if dew point temperature is greater than air temperature comparison = dew_point_temperature_degC > temperature - msg = "dew_point_temperature_degC cannot be greater than temperature." + msg = ("Detected dew_point_temperature_degC values which are greater than corresponding air temperature values." + "\n This implies that the relative humidity would be greater than 100%.") # For pandas Series, ignore NaNs when checking if isinstance(comparison, pd.Series): if comparison.fillna(False).any(): - raise ValueError(msg) + warnings.warn(msg) # For scalar booleans - else: - if comparison: - raise ValueError(msg) + elif comparison: + warnings.warn(msg) # Calculate partial pressure of water vapour water_vapour_press_Pa = _calc_water_vapour_pressure_Pa(air_temperature_degC=temperature, @@ -2436,6 +2437,80 @@ def _calc_water_saturation_vapour_pressure_Pa(air_temperature_degC: Union[float, return E_s_Pa +def calc_rel_humidity_from_dew_point(dew_point_temperature_degC: Union[float, pd.Series], + air_temperature_degC: Union[float, pd.Series] + ) -> Union[float, pd.Series]: + """ + Calculate relative humidity (%) from dew point temperature (degrees Celsius) and air temperature (degrees Celsius). + Note that dew point temperature must be less than or equal to air temperature, otherwise a warning is raised. + + **Calculation method:** + RH = 100 * (E_s(Td) / E_s(T)) + where: + RH = relative humidity (%) + E_s(Td) = saturation vapour pressure at dew point temperature (Td) + E_s(T) = saturation vapour pressure at air temperature (T) + E_s is calculated using the Herman Wobus polynomial approximation as implemented in + `_calc_water_saturation_vapour_pressure_Pa()`. + For a detailed description of the Herman Wobus polynomial approximation see the + 'Vapor Pressure' section at the following link: + https://wahiduddin.net/calc/density_altitude.htm + + :param dew_point_temperature_degC: Dew point temperature in degrees Celsius. + :type dew_point_temperature_degC: float or pandas.Series + :param air_temperature_degC: Air temperature in degrees Celsius. + :type air_temperature_degC: float or pandas.Series + :return: Relative humidity as a percentage. + :rtype: float or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # Calculate relative humidity from scalar values of dew point and air temperature + bw.calc_rel_humidity_from_dew_point(10, 11) + # 93.54469072330612 + + # Calculate relative humidity from series of dew point and air temperature (taken from MERRA-2 reanalysis data) + merra2_node = bw.LoadBrightHub.get_reanalysis('MERRA-2', 53.5, -10.8, '2025-01-01','2025-02-01', nearest_nodes=1, + variables=['Tmp_2m_degC', 'DPTmp_2m_degC'], print_status=True) + rel_humidity = bw.calc_rel_humidity_from_dew_point(merra2_node[1]['DPTmp_2m_degC'], merra2_node[1]['Tmp_2m_degC']) + rel_humidity.head(5) + # Timestamp + # 2025-01-01 00:00:00 71.970262 + # 2025-01-01 01:00:00 72.455810 + # 2025-01-01 02:00:00 72.945025 + # 2025-01-01 03:00:00 73.952574 + # 2025-01-01 04:00:00 74.452697 + # dtype: float64 + """ + # Validate input types + _assert_function_variable_type(dew_point_temperature_degC, (float, int, pd.Series), 'dew_point_temperature_degC') + _assert_function_variable_type(air_temperature_degC, (float, int, pd.Series), 'air_temperature_degC') + # Check index is identical if inputs are pd.Series + _assert_series_index_match(air_temperature_degC, dew_point_temperature_degC, + 'air_temperature_degC', 'dew_point_temperature_degC') + + # Raise error if dew point temperature is greater than air temperature + comparison = dew_point_temperature_degC > air_temperature_degC + msg = ("Detected dew_point_temperature_degC values which are greater than corresponding air temperature values." + "\n This will result in a calculated relative humidity value greater than 100%.") + # For pandas Series, ignore NaNs when checking + if isinstance(comparison, pd.Series): + if comparison.fillna(False).any(): + warnings.warn(msg) + # For scalar booleans + elif comparison: + warnings.warn(msg) + + # Calculate relative humidity + rel_humidity_percent = 100*(_calc_water_saturation_vapour_pressure_Pa(dew_point_temperature_degC) / + _calc_water_saturation_vapour_pressure_Pa(air_temperature_degC)) + rel_humidity_percent.name = 'relative_humidity' + return rel_humidity_percent + + def _calc_water_vapour_pressure_Pa(air_temperature_degC: Optional[Union[float, pd.Series]] = None, rel_humidity_percent: Optional[Union[float, pd.Series]] = None, dew_point_temperature_degC: Optional[Union[float, pd.Series]] = None, diff --git a/tests/test_analyse.py b/tests/test_analyse.py index c71f8051..0f3f7862 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -754,6 +754,39 @@ def test_dist_matrix_by_direction_sector(): assert True +def test_calc_rel_humidity_from_dew_point(): + # test error for dew point greater than temperature + with pytest.raises(ValueError) as except_info: + bw.calc_rel_humidity_from_dew_point(11, 10) + assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." + with pytest.raises(ValueError) as except_info: + bw.calc_rel_humidity_from_dew_point(dew_point_temperature_degC=DATA.T2m.tail(3)+[-1, 0, 1], + air_temperature_degC=DATA.T2m.tail(3)) + assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." + with pytest.raises(ValueError) as except_info: + bw.calc_rel_humidity_from_dew_point(dew_point_temperature_degC=20, + air_temperature_degC=DATA.T2m.tail(3)) + assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." + with pytest.raises(ValueError) as except_info: + bw.calc_rel_humidity_from_dew_point(dew_point_temperature_degC=DATA.T2m.tail(3), + air_temperature_degC=0) + assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." + # test error for Series inputs with different lengths + with pytest.raises(ValueError) as except_info: + bw.calc_rel_humidity_from_dew_point(DATA.T2m.tail(5), DATA.T2m.tail(3)) + assert str(except_info.value) == "air_temperature_degC and dew_point_temperature_degC must have the same dimensions." + # test float/int inputs + assert bw.calc_rel_humidity_from_dew_point(10, 11) - 93.54469072330612 < 1e-3 + assert bw.calc_rel_humidity_from_dew_point(12.1, 12.1) == 100 + # test Series inputs + merra2_node = bw.LoadBrightHub.get_reanalysis('MERRA-2', 53.5, -10.8, '2025-01-01', '2025-02-01', + nearest_nodes=1, variables=['Tmp_2m_degC', 'DPTmp_2m_degC'], + print_status=True) + assert (abs(bw.calc_rel_humidity_from_dew_point(merra2_node[1]['DPTmp_2m_degC'], + merra2_node[1]['Tmp_2m_degC']).head(5).values - + pd.Series([71.97026223, 72.4558096, 72.94502496, 73.9525735, 74.45269691])) < 1e-3).all() + + def test_calc_water_saturation_vapour_pressure_Pa(): # test float input assert abs(bw.analyse.analyse._calc_water_saturation_vapour_pressure_Pa(20) - 2337.237477998109) < 1e-4 From 39686cd508da03fb1b481864d55fd6d6fddacee6 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Tue, 10 Feb 2026 19:42:35 +0000 Subject: [PATCH 30/35] iss #576 allow return of lists and dictionaries from raised child when requested (#578) * iss #576 allow return of lists and dictionaries from raised child when requested * iss #576 bug fix for `calibration_uncertainty` being removed after being created * iss #576 adding tests for list properties now not removed * iss #576 added tests * iss #576 minor fix * iss #576 added changelog entry * iss #576 update changelog entry --------- Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 3 ++- brightwind/load/station.py | 47 +++++++++++++++++++++++++------------- tests/test_station.py | 7 ++++++ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ee66969..431befdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,11 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.5.0] + ### New Features and Enhancements 1. Updated `calc_air_density` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). 2. Added `calc_rel_humidity_from_dew_point` to calculate relative humidity from air temperature and dew point temperature. ([#581](https://github.com/brightwind-dev/brightwind/issues/581)). - +3. Updated `MeasurementStation` function `__get_properties` in order to raise child properties when flattening the dictionary to include sub-lists and sub-dictionaries when raising the following parts of the data model 'mast_properties', 'logger_measurement_config', 'sensor' and 'mounting_arrangement'. This is called by a user with `MeasurementStation.properties` or `MeasurementStation.measurements.properties` ([#576](https://github.com/brightwind-dev/brightwind/issues/576)) ## [2.4.0] diff --git a/brightwind/load/station.py b/brightwind/load/station.py index 889ee7ab..cf8571a3 100644 --- a/brightwind/load/station.py +++ b/brightwind/load/station.py @@ -213,7 +213,7 @@ def _filter_parent_level(dictionary): return parent -def _flatten_dict(dictionary, property_to_bring_up): +def _flatten_dict(dictionary, property_to_bring_up, remove_child_lists=True): """ Bring a child level in a dictionary up to the parent level. @@ -223,6 +223,8 @@ def _flatten_dict(dictionary, property_to_bring_up): :type dictionary: dict :param property_to_bring_up: The child property name to raise up to the parent level. :type property_to_bring_up: str + :param remove_child_lists: Remove child lists from the flattened dictionary. + :type remove_child_lists: bool :return: A list of merged dictionaries :rtype: list(dict) """ @@ -231,11 +233,15 @@ def _flatten_dict(dictionary, property_to_bring_up): for key, value in dictionary.items(): if (type(value) == list) and (key == property_to_bring_up): for item in value: - child = _filter_parent_level(item) + child = item + if remove_child_lists: + child = _filter_parent_level(item) child = _add_prefix(child, property_section=property_to_bring_up) result.append(_merge_two_dicts(parent, child)) if (type(value) == dict) and (key == property_to_bring_up): - child = _filter_parent_level(value) + child = value + if remove_child_lists: + child = _filter_parent_level(value) child = _add_prefix(child, property_section=property_to_bring_up) # return a dictionary and not a list result = _merge_two_dicts(parent, child) @@ -245,11 +251,12 @@ def _flatten_dict(dictionary, property_to_bring_up): return result -def _raise_child(dictionary, child_to_raise): +def _raise_child(dictionary, child_to_raise, remove_child_lists=True): """ :param dictionary: :param child_to_raise: + :param remove_child_lists: :return: """ # FUTURE DEV: ACCOUNT FOR 'DATE_OF_CALIBRATION' WHEN RAISING UP MULTIPLE CALIBRATIONS @@ -259,12 +266,12 @@ def _raise_child(dictionary, child_to_raise): for key, value in dictionary.items(): if (key == child_to_raise) and (value is not None): # Found the key to raise. Flattening dictionary. - return _flatten_dict(dictionary, child_to_raise) + return _flatten_dict(dictionary, child_to_raise, remove_child_lists=remove_child_lists) # didn't find the child to raise. search down through each nested dict or list for key, value in dictionary.items(): if (type(value) == dict) and (value is not None): # 'key' is a dict, looping through it's own keys. - flattened_dicts = _raise_child(value, child_to_raise) + flattened_dicts = _raise_child(value, child_to_raise, remove_child_lists=remove_child_lists) if flattened_dicts: new_dict[key] = flattened_dicts return new_dict @@ -272,7 +279,7 @@ def _raise_child(dictionary, child_to_raise): # 'key' is a list, looping through it's items. temp_list = [] for idx, item in enumerate(value): - flattened_dicts = _raise_child(item, child_to_raise) + flattened_dicts = _raise_child(item, child_to_raise, remove_child_lists=remove_child_lists) if flattened_dicts: if isinstance(flattened_dicts, list): for flat_dict in flattened_dicts: @@ -291,7 +298,7 @@ def _raise_child(dictionary, child_to_raise): 'mast_properties': { 'prefix_separator': '.', 'title_prefix': 'Mast ', - 'keys_to_prefix': ['notes', 'update_at'] + 'keys_to_prefix': ['notes', 'update_at', 'mast_section_geometry'] }, 'vertical_profiler_properties': { 'prefix_separator': '.', @@ -306,7 +313,9 @@ def _raise_child(dictionary, child_to_raise): 'logger_measurement_config': { 'prefix_separator': '.', 'title_prefix': 'Logger ', - 'keys_to_prefix': ['height_m', 'serial_number', 'slope', 'offset', 'sensitivity', 'notes', 'update_at'] + 'keys_to_prefix': [ + 'height_m', 'serial_number', 'slope', 'offset', 'sensitivity', 'notes', 'update_at', 'column_name' + ] }, 'column_name': { 'prefix_separator': '.', @@ -321,8 +330,10 @@ def _raise_child(dictionary, child_to_raise): 'calibration': { 'prefix_separator': '.', 'title_prefix': 'Calibration ', - 'keys_to_prefix': ['slope', 'offset', 'sensitivity', 'report_file_name', 'report_link', - 'uncertainty_k_factor', 'date_from', 'date_to', 'notes', 'update_at'] + 'keys_to_prefix': [ + 'slope', 'offset', 'sensitivity', 'report_file_name', 'report_link', 'uncertainty_k_factor', + 'date_from', 'date_to', 'notes', 'update_at', 'calibration_uncertainty' + ] }, 'calibration_uncertainty': { 'prefix_separator': '.', @@ -489,7 +500,7 @@ def type(self): def __get_properties(self): meas_loc_prop = [] if self.type in ['mast', 'solar']: - meas_loc_prop = _flatten_dict(self.__meas_loc_data_model, property_to_bring_up='mast_properties') + meas_loc_prop = _flatten_dict(self.__meas_loc_data_model, property_to_bring_up='mast_properties', remove_child_lists=False) elif self.type in ['lidar', 'sodar', 'floating_lidar']: meas_loc_prop = _flatten_dict(self.__meas_loc_data_model, property_to_bring_up='vertical_profiler_properties') @@ -823,14 +834,18 @@ def __meas_point_merge(logger_measurement_configs, sensors=None, mounting_arrang def __get_properties(self): meas_props = [] for meas_point in self._meas_data_model: - logger_meas_configs = _raise_child(meas_point, child_to_raise='logger_measurement_config') - calib_raised = _raise_child(meas_point, child_to_raise='calibration') + logger_meas_configs = _raise_child( + meas_point, child_to_raise='logger_measurement_config', remove_child_lists=False + ) + calib_raised = _raise_child(meas_point, child_to_raise='calibration', remove_child_lists=False) if calib_raised is None: sensors = _raise_child(meas_point, child_to_raise='sensor') else: - sensors = _raise_child(calib_raised, child_to_raise='sensor') + sensors = _raise_child(calib_raised, child_to_raise='sensor', remove_child_lists=False) sensors = [sensors_needed for sensors_needed in sensors if sensors_needed['measurement_type_id'] == meas_point['measurement_type_id']] - mounting_arrangements = _raise_child(meas_point, child_to_raise='mounting_arrangement') + mounting_arrangements = _raise_child( + meas_point, child_to_raise='mounting_arrangement', remove_child_lists=False + ) if mounting_arrangements is None: meas_point_merged = self.__meas_point_merge(logger_measurement_configs=logger_meas_configs, diff --git a/tests/test_station.py b/tests/test_station.py index 6d4764c2..7ed43e1f 100644 --- a/tests/test_station.py +++ b/tests/test_station.py @@ -154,6 +154,13 @@ def test_get_table(): def test_properties(): + assert "mast_properties.mast_section_geometry" in MM1.properties + assert "calibration.calibration_uncertainty" in MM1.measurements.properties[0] + assert "logger_measurement_config.column_name" in MM1.measurements.properties[0] + + for measurement in FL1.measurements.properties: + assert "logger_measurement_config.column_name" in measurement + # Check thermometer has correct properties assigned properties = MM1.measurements.properties for measurement in properties: From bb0f82291b858955c3af91b9477c00d436755b0d Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Tue, 10 Feb 2026 22:08:18 +0000 Subject: [PATCH 31/35] Refactored apply_wind_vane_deadband_offset to use _apply_dir_offset_target_orientation (#577) * iss #498 refactor and add tests * iss #498 few updates doctring * iss #498 initial fixes and updates requested * iss #498 fix tests * iss #498 remove NaN to None and bug when columns aren't present * iss #498 updated print statements * iss #498 bug fix in `_apply_dir_offset_target_orientation` * iss #498 including date to logic from `apply_device_orientation_offset` in `apply_wind_vane_deadband_offset` * iss #498 additional test for overlapping periods * iss #498 removed date logic to subfunction * iss #498 updated subfunction name * iss #498 rewording prints * iss #498 put back error * iss #498 updated changelog * iss #498 add sort by name * Update CHANGELOG.md --------- Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 9 +- brightwind/transform/transform.py | 378 +++++++++++++++++------------- tests/test_transform.py | 181 ++++++++++++++ 3 files changed, 405 insertions(+), 163 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 431befdd..bf253e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,12 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.5.0] ### New Features and Enhancements -1. Updated `calc_air_density` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). -2. Added `calc_rel_humidity_from_dew_point` to calculate relative humidity from air temperature and dew point temperature. ([#581](https://github.com/brightwind-dev/brightwind/issues/581)). -3. Updated `MeasurementStation` function `__get_properties` in order to raise child properties when flattening the dictionary to include sub-lists and sub-dictionaries when raising the following parts of the data model 'mast_properties', 'logger_measurement_config', 'sensor' and 'mounting_arrangement'. This is called by a user with `MeasurementStation.properties` or `MeasurementStation.measurements.properties` ([#576](https://github.com/brightwind-dev/brightwind/issues/576)) +1. Updated `calc_air_density()` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa()` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa()`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). +2. Added `calc_rel_humidity_from_dew_point()` to calculate relative humidity from air temperature and dew point temperature. ([#581](https://github.com/brightwind-dev/brightwind/issues/581)). +3. Updated `MeasurementStation` function `__get_properties()` in order to raise child properties when flattening the dictionary to include sub-lists and sub-dictionaries when raising the following parts of the data model 'mast_properties', 'logger_measurement_config', 'sensor' and 'mounting_arrangement'. This is called by a user with `MeasurementStation.properties` or `MeasurementStation.measurements.properties` ([#576](https://github.com/brightwind-dev/brightwind/issues/576)) +4. Fixed two bugs on `apply_device_orientation_offset()` reported table and prints. ([#571](https://github.com/brightwind-dev/brightwind/issues/571)) +5. Fixed bug on `apply_wind_vane_deadband_offset()` reported table and prints. ([#569](https://github.com/brightwind-dev/brightwind/issues/569)) +6. Updated `apply_wind_vane_deadband_offset()` to use same core function than `apply_device_orientation_offset()` for the adjustment. ([#498](https://github.com/brightwind-dev/brightwind/issues/498)) ## [2.4.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index d8b53e49..01d4f424 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1315,7 +1315,8 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re # Depending on what is sent, get wdir properties into a list of properties wdirs_properties = _get_consistent_properties_format(measurements, 'wind_direction') if not wdirs_properties: - raise ValueError('No wind direction measurements found.') + raise ValueError("No wind direction measurements found in the 'measurements' input. " \ + "No deadband offset adjustments can be applied.") # copy the data if needed data = data.copy(deep=True) if inplace is False else data @@ -1324,88 +1325,101 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re # Apply the offset rows = [] - for wdir_prop in wdirs_properties: + col_not_in_data = [] + for i, wdir_prop in enumerate(wdirs_properties): name = wdir_prop['name'] - if name in df.columns: - wdir_in_dataset = True - date_to = wdir_prop.get('date_to') - if date_to is None or date_to == DATE_INSTEAD_OF_NONE: - date_to_txt = 'the end of dataset' - else: - date_to_txt = date_to + if name in df.columns: + date_from, date_to = _resolve_period_boundaries(df, wdirs_properties, i, name) deadband = wdir_prop.get('vane_dead_band_orientation_deg') - date_from = wdir_prop['date_from'] - # Account for a logger offset logger_offset = wdir_prop.get('logger_measurement_config.offset') - offset = deadband - additional_comment_txt = 'to account for deadband' - if logger_offset is not None and logger_offset != 0 and deadband is not None: - offset = offset_wind_direction(float(deadband), offset=-float(logger_offset)) - additional_comment_txt = additional_comment_txt + ' and logger offset' - - if offset: - df[name][date_from:date_to] = \ - offset_wind_direction(df[name][date_from:date_to], - float(offset)) - print('{0} adjusted by {1} degrees from {2} to {3} {4}.\n' - .format(utils.bold(name), utils.bold(str(offset)), - utils.bold(date_from), utils.bold(date_to_txt), additional_comment_txt)) - elif offset == 0: - print('{} has an offset to be applied of 0 from {} to {} {}.\n' - .format(utils.bold(name), utils.bold(date_from), utils.bold(date_to_txt), - additional_comment_txt)) - else: - print('{} has dead_band_orientation of None from {} to {}.\n' - .format(utils.bold(name), utils.bold(date_from), utils.bold(date_to_txt))) height = wdir_prop.get('height_m') - rows.append({ - "Name": name, - "Height [m]": height, - "Vane Dead Band Orientation [deg]": deadband, - "Logger Offset": logger_offset, - "Offset Applied [deg]": offset, - "Date From": date_from, - "Date To": date_to - }) + wdir_in_dataset = True + df[name], applied_results = _apply_dir_offset_target_orientation( + df[name], + logger_offset, + deadband, + date_from, + date_to, + target_orientation_name='dead band orientation', + heights=height, + target_orientation_table_name="Vane Dead Band Orientation [deg]" + ) + rows.append(applied_results) else: - print('{} is not found in data.\n'.format(utils.bold(name))) + col_not_in_data.append(name) + if wdir_in_dataset is False: - print('No wind direction measurement type found in the data.\n') + print('None of the wind direction measurements reported in the "measurements" input is found in the data. ' + 'No deadband offset adjustments can be applied.\n') + if col_not_in_data and wdir_in_dataset: + print( + f"Following wind direction measurement(s) reported in the 'measurements' input not found in the data: " + f"{utils.bold(str(col_not_in_data))}." + ) # if a Series is sent, send back a Series - if type(data) == pd.Series: + if isinstance(data, pd.Series): df = df[df.columns[0]] if return_results_table: - results_df = pd.DataFrame(rows).sort_values( - by=["Height [m]", "Date From"], ascending=[False, True] - ) - results_df['consecutive_group'] = ( - (results_df['Name'] != results_df['Name'].shift()) | - (results_df['Height [m]'] != results_df['Height [m]'].shift()) | - (results_df['Vane Dead Band Orientation [deg]'] != results_df['Vane Dead Band Orientation [deg]'].shift()) | - (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | - (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) - ).cumsum() - - # Group and aggregate consecutive periods - results_table = results_df.groupby([ - 'Name', - 'consecutive_group', - 'Height [m]', - 'Vane Dead Band Orientation [deg]', - 'Logger Offset', - 'Offset Applied [deg]' - ]).agg({ - 'Date From': 'first', - 'Date To': lambda x: None if any(d is None for d in x) else max(x) - }).reset_index(drop=False).drop(columns=['consecutive_group']).set_index("Name").sort_values( - by=["Height [m]", "Date From"], ascending=[False, True] - ) - return df, results_table + if rows: + return df, _aggregate_consecutive_periods( + rows, target_orientation_table_name="Vane Dead Band Orientation [deg]" + ) + else: + return df, None return df +def _aggregate_consecutive_periods(offset_applied_tables, target_orientation_table_name): + """ + Function creates a dataframe from list of tables containing offset applied for each measurement. For each table + neighbouring rows are agreegated into one row if they are consecutive periods that contain the same information + for the following columns: "Name", "Height [m], "Logger Offset", "Offset Applied [deg]" + and target_orientation_table_name. + + The list of offset_applied_tables can be created using _apply_dir_offset_target_orientation function. + + :param offset_applied_tables: List of tables containing offset applied for each measurement. + :type offset_applied_tables: List[pd.DataFrame] + :param target_orientation_table_name: Name of the column on tables representing the target orientation + e.g. "Vane Dead Band Orientation [deg]". + :type target_orientation_table_name: str + :return: DataFrame of offset applied for each measurement, with consecutive periods + merged when appropriate. + :rtype: pd.DataFrame + """ + results_df = pd.concat(offset_applied_tables).sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + results_df['consecutive_group'] = ( + (results_df['Name'] != results_df['Name'].shift()) | + (results_df['Height [m]'] != results_df['Height [m]'].shift()) | + (results_df[target_orientation_table_name] != results_df[target_orientation_table_name].shift()) | + (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | + (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) + ).cumsum() + + # Group and aggregate consecutive periods + results_table = results_df.groupby([ + 'Name', + 'consecutive_group', + 'Height [m]', + target_orientation_table_name, + 'Logger Offset', + 'Offset Applied [deg]' + ], dropna=False).agg({ + 'Date From': 'first', + 'Date To': lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(drop=False).drop(columns=['consecutive_group']).sort_values( + by=["Height [m]", "Date From", "Name"], ascending=[False, True, True] + ).set_index("Name") + results_table = results_table.fillna({ + 'Offset Applied [deg]': 0 + }) + return results_table + + def _selective_avg(wspd1, wspd2, wdir, boom_dir1, boom_dir2, inflow_lower1, inflow_higher1, inflow_lower2, inflow_higher2, sector_width): # duplicate threshold values into lists which are the same length as other inputs @@ -1749,6 +1763,10 @@ def apply_device_orientation_offset( measurements = measurement_station.measurements wdirs_properties = _get_consistent_properties_format(measurements, 'wind_direction') + if not wdirs_properties: + raise ValueError("No wind direction measurements found in the 'measurement_station' input. " \ + "No device orientation offset adjustments can be applied.") + measurement_station_items = list(measurement_station) # copy the data if needed data = data.copy(deep=True) if inplace is False else data @@ -1776,23 +1794,10 @@ def apply_device_orientation_offset( # Apply the offset for i, wdir_prop in enumerate(wdirs_properties): name = wdir_prop['name'] + if name in df.columns: - date_to = wdir_prop.get('date_to') - # If the last logger properties date to has been explicitly set as the last timestamp of the dataset, - # set it to None. This avoids missing this timestamp due to [date_from, date_to) logic - if date_to is not None: - if pd.to_datetime(date_to) >= df.index[-1]: - date_to = None - # If [date_from, date_to) convention has not been used, we force this convention by setting - # date_to to the date_from of the next logger property - if i < len(wdirs_properties) - 1: - if wdirs_properties[i+1].get('name') == name: - next_date_from = wdirs_properties[i+1].get('date_from') - if next_date_from != date_to: - date_to = next_date_from - date_from = wdir_prop.get('date_from') - date_from = (df.index[0].strftime('%Y-%m-%dT%H:%M:%S') - if date_from is None or date_from == DATE_INSTEAD_OF_NONE else date_from) + date_from, date_to = _resolve_period_boundaries(df, wdirs_properties, i, name) + logger_offset = wdir_prop.get('logger_measurement_config.offset') for j, device_properties in enumerate(measurement_station): meas_station_data_model_from = device_properties.get('date_from') @@ -1839,20 +1844,15 @@ def apply_device_orientation_offset( apply_offset_to = date_to_tmp if date_to_tmp is not None else meas_station_data_model_to else: apply_offset_to = min(date_to_tmp, meas_station_data_model_to) - df[name] = _apply_dir_offset_target_orientation( + height = wdir_prop.get('height_m') + + df[name], applied_results = _apply_dir_offset_target_orientation( df[name], logger_offset, device_orientation_deg, apply_offset_from, apply_offset_to, - target_orientation_name='device orientation') + target_orientation_name='device orientation', heights=height, + target_orientation_table_name="Device Orientation [deg]" + ) - height = wdir_prop.get('height_m') - rows.append({ - "Name": name, - "Height [m]": height, - "Device Orientation [deg]": device_orientation_deg, - "Logger Offset": logger_offset, - "Offset Applied [deg]": offset_wind_direction(device_orientation_deg, - logger_offset), - "Date From": apply_offset_from, - "Date To": apply_offset_to - }) + rows.append(applied_results) else: wdir_not_in_dataset = True col_not_in_data.append(name) @@ -1860,14 +1860,15 @@ def apply_device_orientation_offset( if wdir_not_in_dataset: indexes = np.unique(col_not_in_data, return_index=True)[1] col_not_in_data = [col_not_in_data[index] for index in sorted(indexes)] - print_text = 'Following wind direction measurement(s) not found in the data' + print_text = "Following wind direction measurement(s) reported in the 'measurement_station' input " \ + "not found in the data" if wdir_cols: print(print_text + ' for the requested `wdir_cols`: {}.'.format(utils.bold(str(col_not_in_data)))) else: print(print_text + ': {}.'.format(utils.bold(str(col_not_in_data)))) if col_not_in_datamodel: - print('No device orientation offset applied to following requested measurement(s) as no wind direction ' - 'measurement type found in `meas_station_data_models` for these: {}.' + print('No device orientation offset applied to following `wdir_cols` requested measurement(s) ' \ + 'as no wind direction measurement type found in `measurement_station` input for these: {}.' .format(utils.bold(str(col_not_in_datamodel)))) # if a Series is sent, send back a Series if isinstance(data, pd.Series): @@ -1875,35 +1876,55 @@ def apply_device_orientation_offset( data.update(df) if return_results_table: - results_df = pd.DataFrame(rows).sort_values( - by=["Height [m]", "Date From"], ascending=[False, True] - ) - results_df['consecutive_group'] = ( - (results_df['Name'] != results_df['Name'].shift()) | - (results_df['Height [m]'] != results_df['Height [m]'].shift()) | - (results_df['Device Orientation [deg]'] != results_df['Device Orientation [deg]'].shift()) | - (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | - (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) - ).cumsum() - - # Group and aggregate consecutive periods with the same device orientation, logger offset and applied offset. - results_table = results_df.groupby([ - 'Name', - 'consecutive_group', - 'Height [m]', - 'Device Orientation [deg]', - 'Logger Offset', - 'Offset Applied [deg]' - ]).agg({ - 'Date From': 'first', - 'Date To': lambda x: None if any(d is None for d in x) else max(x) - }).reset_index(drop=False).drop(columns=['consecutive_group']).set_index("Name").sort_values( - by=["Height [m]", "Date From"], ascending=[False, True] - ) - return df, results_table + if rows: + return df, _aggregate_consecutive_periods(rows, target_orientation_table_name="Device Orientation [deg]") + else: + return df, None return df +def _resolve_period_boundaries(df, wdirs_properties, current_index, name): + """ + Normalize logger period time boundaries to enforce [date_from, date_to) half-open interval logic. + + This ensures data points are not corrected twice by making the earlier logger property period + end exactly when the next entry starts. + + :param df: Wind measurement dataframe with datetime index + :type df: pd.DataFrame + :param wdirs_properties: Consistent list of properties for the wind_direction + :type wdirs_properties: list + :param current_index: Index of wind direction property currently being processed + :type current_index: int + :param name: Wind direction name + :type name: str + :return: Tuple of (date_from, date_to) as ISO format strings. date_to may be None to indicate + the period extends to the end of the dataset. + :rtype: Tuple[str, str] + """ + date_to = wdirs_properties[current_index].get('date_to') + + # If the last logger properties date to has been explicitly set as the last timestamp of the dataset, + # set it to None. This avoids missing this timestamp due to [date_from, date_to) logic + if date_to is not None: + if pd.to_datetime(date_to) >= df.index[-1]: + date_to = None + + # If [date_from, date_to) convention has not been used, we force this convention by setting + # date_to to the date_from of the next logger property + if current_index < len(wdirs_properties) - 1: + if wdirs_properties[current_index+1].get('name') == name: + next_date_from = wdirs_properties[current_index + 1].get('date_from') + if next_date_from != date_to: + date_to = next_date_from + + date_from = wdirs_properties[current_index].get('date_from') + date_from = (df.index[0].strftime('%Y-%m-%dT%H:%M:%S') + if date_from is None or date_from == DATE_INSTEAD_OF_NONE else date_from) + + return date_from, date_to + + def _check_vertical_profiler_properties_overlap(measurement_station, df): """ Checks if in vertical_profiler_properties there are any overlapping @@ -1958,13 +1979,18 @@ def _check_vertical_profiler_properties_overlap(measurement_station, df): return False -def _apply_dir_offset_target_orientation(wdir_data, logger_offset, target_orientation, apply_offset_from, - apply_offset_to, target_orientation_name): +def _apply_dir_offset_target_orientation( + wdir_data, logger_offset, target_orientation, apply_offset_from, apply_offset_to, target_orientation_name, + heights, target_orientation_table_name + ): """ Function to apply the required offset to the wind direction data based on the logger offset and a target orientation. Note that if `wdir_data` is a DataFrame, the adjustment derived from `logger_offset` and `target_orientation` is applied to all columns. + + Function returns also a DataFrame containing the target orientation, logger orientation and offset applied + for each relevant time period. This function uses the brightwind 'offset_wind_direction()' function to apply the actual adjustment to the wind direction data. @@ -1979,26 +2005,35 @@ def _apply_dir_offset_target_orientation(wdir_data, logger_offset, target_orient Date ranges are considered as [from, to) where 'from' is inclusive and 'to' is exclusive. - :param wdir_data: The wind direction data time series. - :type wdir_data: pd.Series or pd.DataFrame - :param logger_offset: The logger offset value in degrees for the input wind direction data. - :type logger_offset: float - :param target_orientation: The target orientation value in degrees. - :type target_orientation: float - :param apply_offset_from: The date to apply the offset from. - :type apply_offset_from: str | datetime.datetime | pd.Timestamp - :param apply_offset_to: The date to apply the offset to, treated in and exclusive manner. - :type apply_offset_to: str | datetime.datetime | pd.Timestamp - :param target_orientation_name: The target orientation name to use for the print statements. - e.g 'device orientation' or 'deadband orientation' - :type target_orientation_name: str + :param wdir_data: The wind direction data time series. + :type wdir_data: pd.Series or pd.DataFrame + :param logger_offset: The logger offset value in degrees for the input wind direction data. + :type logger_offset: float + :param target_orientation: The target orientation value in degrees. + :type target_orientation: float + :param apply_offset_from: The date to apply the offset from. + :type apply_offset_from: str | datetime.datetime | pd.Timestamp + :param apply_offset_to: The date to apply the offset to, treated in and exclusive manner. + :type apply_offset_to: str | datetime.datetime | pd.Timestamp + :param target_orientation_name: The target orientation name to use for the print statements. + e.g 'device orientation' or 'deadband orientation' + :type target_orientation_name: str + :param heights: The height(s) corresponding to the measurement(s) in + wdir_data . + :type heights: list | float + :param target_orientation_table_name: The target orientation name to use for the column of the returned results + DataFrame. + :type target_orientation_table_name: str + :return: A tuple of the data and a DataFrame with the appropriate offset applied + in addition to a table of the offset information. + :rtype: Tuple[pd.DataFrame, pd.DataFrame] """ offset = target_orientation - wdir_names = list(wdir_data.columns) if isinstance(wdir_data, pd.DataFrame) else wdir_data.name + wdir_names = list(wdir_data.columns) if isinstance(wdir_data, pd.DataFrame) else [wdir_data.name] additional_comment_txt = 'to account for {}'.format(target_orientation_name) - if apply_offset_to is None: + if apply_offset_to is None or apply_offset_to == DATE_INSTEAD_OF_NONE: to_text = "end of data" mask = (wdir_data.index >= pd.Timestamp(apply_offset_from)) else: @@ -2011,27 +2046,50 @@ def _apply_dir_offset_target_orientation(wdir_data, logger_offset, target_orient apply_offset_to_inclusive = wdir_data.index[idx_pos - 1].strftime('%Y-%m-%dT%H:%M:%S') to_text = f"{apply_offset_to} (exclusive)" if apply_offset_from > apply_offset_to_inclusive: - return wdir_data + return wdir_data, pd.DataFrame([]) if logger_offset is not None and logger_offset != 0 and target_orientation is not None: offset = offset_wind_direction(float(target_orientation), offset=-float(logger_offset)) additional_comment_txt = additional_comment_txt + ' and logger offset' - if offset: + if offset: # Apply offset only to the masked data wdir_data.loc[mask] = offset_wind_direction(wdir_data.loc[mask], float(offset)) + print_statement = '{0} adjusted by {1} degrees from {2} to {3} {4}.\n' + if logger_offset is None: + print_statement = print_statement.strip("\n") + ' The logger offset value is set as None.\n' + + print(print_statement.format(utils.bold(", ".join(wdir_names)), utils.bold(str(offset)), + utils.bold(str(apply_offset_from)), utils.bold(to_text), + additional_comment_txt)) + elif offset == 0: + print_statement = '{0} has an offset to be applied of 0 degrees from {1} to {2} {3}.\n' + if logger_offset is None: + print_statement = print_statement.strip("\n") + ' The logger offset value is set as None.\n' + print(print_statement.format(utils.bold(str(", ".join(wdir_names))), utils.bold(str(apply_offset_from)), + utils.bold(to_text), + additional_comment_txt)) - print('{0} adjusted by {1} degrees from {2} to {3} {4}.\n' - .format(utils.bold(str(wdir_names)), utils.bold(str(offset)), - utils.bold(str(apply_offset_from)), utils.bold(to_text), - additional_comment_txt)) - elif offset == 0: - print('{0} has an offset to be applied of 0 degrees from {1} to {2} {3}.\n' - .format(utils.bold(str(wdir_names)), utils.bold(str(apply_offset_from)), - utils.bold(to_text), - additional_comment_txt)) - else: - print('{0} has {1} as None from {2} to {3}.\n' - .format(utils.bold(str(wdir_names)), target_orientation_name, - utils.bold(str(apply_offset_from)), utils.bold(to_text))) - - return wdir_data + else: + if logger_offset is None: + print('{0} has {1} as None and logger offset as None from {2} to {3}. No adjustment applied.\n' + .format(utils.bold(str(", ".join(wdir_names))), target_orientation_name, + utils.bold(str(apply_offset_from)), utils.bold(to_text))) + else: + print('{0} has {1} as None from {2} to {3}. No adjustment applied.\n' + .format(utils.bold(str(", ".join(wdir_names))), target_orientation_name, + utils.bold(str(apply_offset_from)), utils.bold(to_text))) + + rows = [] + if not isinstance(heights, (list, tuple, np.ndarray)): + heights = [heights] * len(wdir_names) + for (wdir_name, height) in zip(wdir_names, heights): + rows.append({ + "Name": wdir_name, + "Height [m]": height, + target_orientation_table_name: target_orientation, + "Logger Offset": logger_offset, + "Offset Applied [deg]": offset, + "Date From": apply_offset_from, + "Date To": apply_offset_to + }) + return wdir_data, pd.DataFrame(rows) diff --git a/tests/test_transform.py b/tests/test_transform.py index fe9b1c2b..d7acedd6 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -182,6 +182,187 @@ def test_apply_wind_vane_dead_band_offset(): data['Dir78mS'].fillna(0).round(10)).all() +def test_apply_wind_vane_dead_band_offset_table_returned(): + data = bw.load_csv(bw.demo_datasets.demo_data) + data['Dir78_test'] = data['Dir78mS'].copy() + data['Dir58_test'] = data['Dir58mS'].copy() + + test_meas_config_dict = { + 'Dir78mS': [{'name': 'Dir78mS', + 'measurement_type_id': 'wind_direction', 'height_m': 78.0, + 'logger_measurement_config.slope': 0.07263, + 'logger_measurement_config.offset': 130.87, + 'date_from': '2025-02-20T00:10:00', + 'date_to': None, + 'vane_dead_band_orientation_deg': None}], + 'Dir78_test': [{'name': 'Dir78_test', + 'measurement_type_id': 'wind_direction', + 'height_m': 78.0, + 'logger_measurement_config.slope': 0.07263, + 'logger_measurement_config.offset': None, + 'date_from': '2025-02-20T00:10:00', + 'date_to': None, + 'vane_dead_band_orientation_deg': None}], + 'Dir58mS': [{'name': 'Dir58mS', + 'measurement_type_id': 'wind_direction', + 'height_m': 58.0, + 'logger_measurement_config.slope': 0.07262, + 'logger_measurement_config.offset': 127.75, + 'date_from': '2025-02-20T00:10:00', + 'date_to': None, + 'vane_dead_band_orientation_deg': 127.75}], + 'Dir58_test': [{'name': 'Dir58_test', + 'measurement_type_id': 'wind_direction', + 'height_m': 58.0, + 'logger_measurement_config.slope': 0.07262, + 'logger_measurement_config.offset': None, + 'date_from': '2025-02-20T00:10:00', + 'date_to': None, + 'vane_dead_band_orientation_deg': 127.75}], + 'Dir38mS': [{'name': 'Dir38mS', + 'measurement_type_id': 'wind_direction', + 'height_m': 38.0, + 'logger_measurement_config.slope': 22.5, + 'logger_measurement_config.offset': 36.72, + 'date_from': '2025-02-20T00:10:00', + 'date_to': None, + 'vane_dead_band_orientation_deg': 2}] + } + expected_table = pd.DataFrame([{'Name': 'Dir78_test', + 'Height [m]': 78.0, + 'Vane Dead Band Orientation [deg]': None, + 'Logger Offset': None, + 'Offset Applied [deg]': 0.0, + 'Date From': '2025-02-20T00:10:00', + 'Date To': None}, + {'Name': 'Dir78mS', + 'Height [m]': 78.0, + 'Vane Dead Band Orientation [deg]': None, + 'Logger Offset': 130.87, + 'Offset Applied [deg]': 0.0, + 'Date From': '2025-02-20T00:10:00', + 'Date To': None}, + {'Name': 'Dir58_test', + 'Height [m]': 58.0, + 'Vane Dead Band Orientation [deg]': 127.75, + 'Logger Offset': None, + 'Offset Applied [deg]': 127.75, + 'Date From': '2025-02-20T00:10:00', + 'Date To': None}, + {'Name': 'Dir58mS', + 'Height [m]': 58.0, + 'Vane Dead Band Orientation [deg]': 127.75, + 'Logger Offset': 127.75, + 'Offset Applied [deg]': 0.0, + 'Date From': '2025-02-20T00:10:00', + 'Date To': None}, + {'Name': 'Dir38mS', + 'Height [m]': 38.0, + 'Vane Dead Band Orientation [deg]': 2.0, + 'Logger Offset': 36.72, + 'Offset Applied [deg]': 325.28, + 'Date From': '2025-02-20T00:10:00', + 'Date To': None}] + ).set_index('Name') + + data, table = bw.apply_wind_vane_deadband_offset(data, test_meas_config_dict, return_results_table=True) + + pd.testing.assert_frame_equal(table, expected_table) + + # Test for correct behaviour in overlapping periods in measurement config + test_meas_config_dict['Dir78mS'].append( + {'name': 'Dir78mS', + 'measurement_type_id': 'wind_direction', 'height_m': 78.0, + 'logger_measurement_config.slope': 0.07263, + 'logger_measurement_config.offset': 132.87, + 'date_from': '2016-10-20T08:10:00', + 'date_to': None, + 'vane_dead_band_orientation_deg': 3} + ) + test_meas_config_dict['Dir78mS'][0]['date_from'] = '2016-10-20T00:10:00' + test_meas_config_dict['Dir78mS'][0]['date_to'] = '2016-10-20T08:10:00' + test_meas_config_dict['Dir78mS'][0]['vane_dead_band_orientation_deg'] = 1 + + data, table = bw.apply_wind_vane_deadband_offset(data, test_meas_config_dict, return_results_table=True) + + assert np.allclose(data.loc['2016-10-20T08:00:00', "Dir78mS"], 255.11) + + +def test_apply_device_orientation_offset_table_returned(): + fl1 = bw.MeasurementStation(bw.demo_datasets.floating_lidar_demo_iea43_wra_data_model_v1_3) + data_model = fl1.data_model + data_model['measurement_point'][20]['logger_measurement_config'][0]['offset'] = None + data_model['measurement_point'][20]['logger_measurement_config'][1]['offset'] = None + data_model['measurement_point'][20]['logger_measurement_config'][2]['offset'] = None + data_model = { + "author": "Brighthub", + "organisation": "Brightwind", + "date": "2025-03-26", + "version": "1.3.0-2024.03", + "measurement_location": [data_model] + } + data = bw.load_csv(bw.demo_datasets.demo_floating_lidar_data) + fll_test = bw.MeasurementStation(data_model) + fll_test[0]['device_orientation_deg'] = None + _, table = bw.apply_device_orientation_offset(data, fll_test, return_results_table=True) + expected_table = pd.DataFrame([{'Name': 'Dir_50m', + 'Height [m]': 50, + 'Device Orientation [deg]': None, + 'Logger Offset': None, + 'Offset Applied [deg]': 0.0, + 'Date From': '2012-10-23T13:10:00', + 'Date To': '2012-11-15T13:50:00'}, + {'Name': 'Dir_50m', + 'Height [m]': 50, + 'Device Orientation [deg]': None, + 'Logger Offset': None, + 'Offset Applied [deg]': 0.0, + 'Date From': '2012-11-15T13:50:00', + 'Date To': '2012-11-23T12:10:00'}, + {'Name': 'Dir_50m', + 'Height [m]': 50, + 'Device Orientation [deg]': 265.0, + 'Logger Offset': None, + 'Offset Applied [deg]': 265.0, + 'Date From': '2012-11-23T12:10:00', + 'Date To': '2013-10-08T14:00:00'}, + {'Name': 'Dir_50m', + 'Height [m]': 50, + 'Device Orientation [deg]': 265.0, + 'Logger Offset': None, + 'Offset Applied [deg]': 265.0, + 'Date From': '2013-10-08T14:00:00', + 'Date To': None}, + {'Name': 'Dir_40m', + 'Height [m]': 40, + 'Device Orientation [deg]': None, + 'Logger Offset': 170.0, + 'Offset Applied [deg]': 0.0, + 'Date From': '2012-10-23T13:10:00', + 'Date To': '2012-11-15T13:50:00'}, + {'Name': 'Dir_40m', + 'Height [m]': 40, + 'Device Orientation [deg]': None, + 'Logger Offset': 165.0, + 'Offset Applied [deg]': 0.0, + 'Date From': '2012-11-15T13:50:00', + 'Date To': '2012-11-23T12:10:00'}, + {'Name': 'Dir_40m', + 'Height [m]': 40, + 'Device Orientation [deg]': 265.0, + 'Logger Offset': 165.0, + 'Offset Applied [deg]': 100.0, + 'Date From': '2012-11-23T12:10:00', + 'Date To': '2013-10-08T14:00:00'}, + {'Name': 'Dir_40m', + 'Height [m]': 40, + 'Device Orientation [deg]': 265.0, + 'Logger Offset': 262.0, + 'Offset Applied [deg]': 3.0, + 'Date From': '2013-10-08T14:00:00', + 'Date To': None}]).set_index('Name') + pd.testing.assert_frame_equal(table, expected_table) + def test_apply_device_orientation_offset(): actual_series_result = bw.apply_device_orientation_offset( From c4f97416825e2b3427c1e688e161beaaaac2d41e Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Thu, 12 Feb 2026 16:20:48 +0000 Subject: [PATCH 32/35] iss #574 applying adjustment to relevant associated columns (#584) * iss #574 applying adjustment to relevant associated columns * iss #574 applying also to apply_device_orientation_offset * iss #574 fix for difference in column names * iss #574 add changelog entry * iss #574 added test * iss #574 fixed broken test * iss #574 update structure, added optional argument and docstring * iss #574 add gust to be adjusted * iss #574 reformat changelog * iss #574 change default to False * iss #574 change default behaviour * iss #574 udate docstring with gust * iss #574 fix test following default change --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 9 +- brightwind/transform/transform.py | 256 +++++++++++++++++------------- tests/test_transform.py | 73 +++++++++ 3 files changed, 229 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf253e01..a30c5e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,14 @@ Additional labels for pre-release and build metadata are available as extensions 1. Updated `calc_air_density()` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa()` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa()`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). 2. Added `calc_rel_humidity_from_dew_point()` to calculate relative humidity from air temperature and dew point temperature. ([#581](https://github.com/brightwind-dev/brightwind/issues/581)). 3. Updated `MeasurementStation` function `__get_properties()` in order to raise child properties when flattening the dictionary to include sub-lists and sub-dictionaries when raising the following parts of the data model 'mast_properties', 'logger_measurement_config', 'sensor' and 'mounting_arrangement'. This is called by a user with `MeasurementStation.properties` or `MeasurementStation.measurements.properties` ([#576](https://github.com/brightwind-dev/brightwind/issues/576)) -4. Fixed two bugs on `apply_device_orientation_offset()` reported table and prints. ([#571](https://github.com/brightwind-dev/brightwind/issues/571)) -5. Fixed bug on `apply_wind_vane_deadband_offset()` reported table and prints. ([#569](https://github.com/brightwind-dev/brightwind/issues/569)) -6. Updated `apply_wind_vane_deadband_offset()` to use same core function than `apply_device_orientation_offset()` for the adjustment. ([#498](https://github.com/brightwind-dev/brightwind/issues/498)) +4. Updated `apply_wind_vane_deadband_offset()` to use same core function than `apply_device_orientation_offset()` for the adjustment. ([#498](https://github.com/brightwind-dev/brightwind/issues/498)) +5. Updated `apply_device_orientation_offset()` and `apply_wind_vane_deadband_offset()` in order to also apply directional adjustment to related columns (for 'avg', 'min', 'max' and 'gust' statistic types) in addition to the main variable itself. ([#574](https://github.com/brightwind-dev/brightwind/issues/574)). +### Bug Fixes +1. Fixed two bugs on `apply_device_orientation_offset()` reported table and prints. ([#571](https://github.com/brightwind-dev/brightwind/issues/571)) +2. Fixed bug on `apply_wind_vane_deadband_offset()` reported table and prints. ([#569](https://github.com/brightwind-dev/brightwind/issues/569)) +--- ## [2.4.0] ### New Features and Enhancements diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 01d4f424..ccf7517e 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1242,7 +1242,9 @@ def offset_wind_direction(wdir, offset: float): return wdir.add(offset).apply(utils._range_0_to_360) -def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_results_table=False): +def apply_wind_vane_deadband_offset( + data, measurements, inplace=False, return_results_table=False, apply_to_related_statistics=False + ): """ Automatically apply deadband offsets of the wind vanes to the timeseries data. The deadband orientation information for each wind direction measurement and time period is contained in the measurements @@ -1261,23 +1263,35 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re This function accounts for this adjustment. - :param data: Timeseries data. - :type data: pd.DataFrame or pd.Series - :param measurements: Measurement information extracted from a WRA Data Model using bw.MeasurementStation - :type measurements: list or dict or _Measurements - :param inplace: If 'inplace' is True, the original direction data, contained in 'data', will be - modified and replaced with the adjusted direction data. If 'inplace' is False, the - original data will not be touched and instead a new DataFrame containing the - adjusted direction data is created. To store this adjusted direction data, please - ensure it is assigned to a new variable. - :type inplace: bool - :param return_results_table: Optional key to return a dataframe containing deadband offset, logger offset and - applied offset for each directional sensor and the time period it is relevant for. - :type return_results_table: pd.DataFrame - :return: Data with adjusted wind direction by the deadband orientation, or where - return_results_table is specified, a tuple of the data and a DataFrame of - deadband offsets. - :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] + :param data: Timeseries data. + :type data: pd.DataFrame or pd.Series + :param measurements: Measurement information extracted from a WRA Data Model using + bw.MeasurementStation + :type measurements: list or dict or _Measurements + :param inplace: If 'inplace' is True, the original direction data, contained in 'data', will be + modified and replaced with the adjusted direction data. If 'inplace' is False, + the original data will not be touched and instead a new DataFrame containing the + adjusted direction data is created. To store this adjusted direction data, + please ensure it is assigned to a new variable. + :type inplace: bool + :param return_results_table: Optional key to return a dataframe containing deadband offset, logger offset + and applied offset for each directional sensor and the time period it is + relevant for. + :type return_results_table: pd.DataFrame + :param apply_to_related_statistics: If True, apply the adjustment to related statistics (e.g. if Dir60mS is + adjusted, also adjust Dir60mS_max, Dir60mS_min and Dir60mS_gust). + If False, only apply the adjustment to the specific wind direction properties. + If True then the function expects the column name convention where the average + has nothing appended, max is appended with '_max', min is appended with '_min' + and gust is appended with '_gust'. + If the column name convention is different, set this parameter to False and the + adjustment will only be applied to the specific wind direction properties or + rename your data columns. Defaults to False. + :type apply_to_related_statistics: bool + :return: Data with adjusted wind direction by the deadband orientation, or where + return_results_table is specified, a tuple of the data and a DataFrame of + deadband offsets. + :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] **Example usage** :: @@ -1315,8 +1329,8 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re # Depending on what is sent, get wdir properties into a list of properties wdirs_properties = _get_consistent_properties_format(measurements, 'wind_direction') if not wdirs_properties: - raise ValueError("No wind direction measurements found in the 'measurements' input. " \ - "No deadband offset adjustments can be applied.") + raise ValueError("No wind direction measurements found in the 'measurements' input. " + "No deadband offset adjustments can be applied.") # copy the data if needed data = data.copy(deep=True) if inplace is False else data @@ -1328,27 +1342,35 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re col_not_in_data = [] for i, wdir_prop in enumerate(wdirs_properties): name = wdir_prop['name'] - - if name in df.columns: - date_from, date_to = _resolve_period_boundaries(df, wdirs_properties, i, name) - deadband = wdir_prop.get('vane_dead_band_orientation_deg') - logger_offset = wdir_prop.get('logger_measurement_config.offset') - height = wdir_prop.get('height_m') - wdir_in_dataset = True - df[name], applied_results = _apply_dir_offset_target_orientation( - df[name], - logger_offset, - deadband, - date_from, - date_to, - target_orientation_name='dead band orientation', - heights=height, - target_orientation_table_name="Vane Dead Band Orientation [deg]" - ) - rows.append(applied_results) + if not apply_to_related_statistics: + associated_statistics = [name] else: - col_not_in_data.append(name) - + # This assumed variable naming is based on what BrightHub uses + associated_statistics = [ + name if prop["statistic_type_id"] == "avg" else f"{name}_{prop['statistic_type_id']}" + for prop in wdir_prop["logger_measurement_config.column_name"] + if prop["statistic_type_id"] in ["avg", "max", "min", "gust"] + ] + for var_name in associated_statistics: + if var_name in df.columns: + date_from, date_to = _resolve_period_boundaries(df, wdirs_properties, i, name) + deadband = wdir_prop.get('vane_dead_band_orientation_deg') + logger_offset = wdir_prop.get('logger_measurement_config.offset') + height = wdir_prop.get('height_m') + wdir_in_dataset = True + df[var_name], applied_results = _apply_dir_offset_target_orientation( + df[var_name], + logger_offset, + deadband, + date_from, + date_to, + target_orientation_name='dead band orientation', + heights=height, + target_orientation_table_name="Vane Dead Band Orientation [deg]" + ) + rows.append(applied_results) + else: + col_not_in_data.append(var_name) if wdir_in_dataset is False: print('None of the wind direction measurements reported in the "measurements" input is found in the data. ' @@ -1430,7 +1452,7 @@ def _selective_avg(wspd1, wspd2, wdir, boom_dir1, boom_dir2, # if boom 1 'inflow' sector overlaps with 0/360 if ((boom_dir1 + 180) % 360) >= (360 - (sector_width/2)) or ((boom_dir1 + 180) % 360) <= (sector_width/2): - # many nested if statments follow, all within one mapped lambda function + # many nested if statements follow, all within one mapped lambda function sel_avg = list(map(lambda spd1,spd2,Dir,inflowlow1,inflowhigh1,inflowlow2,inflowhigh2: # if one value is Nan, use the other one spd2 if (np.isnan(spd1)==True) else (spd1 if np.isnan(spd2)==True @@ -1686,7 +1708,8 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals def apply_device_orientation_offset( - data, measurement_station, wdir_cols=[], inplace=False, return_results_table=False + data, measurement_station, wdir_cols=[], inplace=False, return_results_table=False, + apply_to_related_statistics=False ): """ Applies a device orientation offset to wind direction data from remote sensing devices @@ -1730,6 +1753,18 @@ def apply_device_orientation_offset( :param return_results_table: If True, returns a DataFrame containing the device orientation, logger orientation and offset applied for each relevant time period. :type return_results_table: bool, optional + :param apply_to_related_statistics: If True, apply the adjustment to related statistics (e.g. if Dir60mS is + adjusted, also adjust Dir60mS_max, Dir60mS_min and Dir60mS_gust). + If False, only apply the adjustment to the specific wind direction + properties. + If True then the function expects the column name convention where the + average has nothing appended, max is appended with '_max', min is + appended with '_min' and gust is appended with '_gust'. + If the column name convention is different, set this + parameter to False and the adjustment will only be applied to the + specific wind direction properties or rename your data columns. + Defaults to False. + :type apply_to_related_statistics: bool :return: Data with wind direction adjusted by the orientation offset. :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] @@ -1764,8 +1799,8 @@ def apply_device_orientation_offset( measurements = measurement_station.measurements wdirs_properties = _get_consistent_properties_format(measurements, 'wind_direction') if not wdirs_properties: - raise ValueError("No wind direction measurements found in the 'measurement_station' input. " \ - "No device orientation offset adjustments can be applied.") + raise ValueError("No wind direction measurements found in the 'measurement_station' input. " + "No device orientation offset adjustments can be applied.") measurement_station_items = list(measurement_station) # copy the data if needed @@ -1794,81 +1829,90 @@ def apply_device_orientation_offset( # Apply the offset for i, wdir_prop in enumerate(wdirs_properties): name = wdir_prop['name'] - - if name in df.columns: - date_from, date_to = _resolve_period_boundaries(df, wdirs_properties, i, name) - - logger_offset = wdir_prop.get('logger_measurement_config.offset') - for j, device_properties in enumerate(measurement_station): - meas_station_data_model_from = device_properties.get('date_from') - meas_station_data_model_from = (df.index[0].strftime('%Y-%m-%dT%H:%M:%S') if - meas_station_data_model_from is None or meas_station_data_model_from == - DATE_INSTEAD_OF_NONE else meas_station_data_model_from) - meas_station_data_model_to = device_properties.get('date_to') - # If the last logger properties date to has been explicitly set as the last timestamp of the dataset, - # set it to None. This avoids missing this timestamp due to [date_from, date_to) logic - if meas_station_data_model_to is not None: - if pd.to_datetime(meas_station_data_model_to) >= df.index[-1]: - meas_station_data_model_to = None - # If [date_from, date_to) convention has not been used, we force this convention by setting - # meas_station_data_model_to to the next_meas_station_data_model_from of the next - # measurement_station property. - if j < len(measurement_station_items) - 1: - next_meas_station_data_model_from = measurement_station[j+1].get('date_from') - if next_meas_station_data_model_from != meas_station_data_model_to: - meas_station_data_model_to = next_meas_station_data_model_from - - if date_to is None or date_to == DATE_INSTEAD_OF_NONE: - date_to_tmp = meas_station_data_model_to - else: - date_to_tmp = date_to + if not apply_to_related_statistics: + associated_statistics = [name] + else: + # This assumed variable naming is based on what BrightHub uses + associated_statistics = [ + name if prop["statistic_type_id"] == "avg" else f"{name}_{prop['statistic_type_id']}" + for prop in wdir_prop["logger_measurement_config.column_name"] + if prop["statistic_type_id"] in ["avg", "max", "min", "gust"] + ] + for var_name in associated_statistics: + if var_name in df.columns: + date_from, date_to = _resolve_period_boundaries(df, wdirs_properties, i, name) - date_range_overlaps = False - if meas_station_data_model_to is None and date_to_tmp is None: - date_range_overlaps = True - elif meas_station_data_model_to is None: - date_range_overlaps = date_to_tmp is None or date_to_tmp >= meas_station_data_model_from - elif date_to_tmp is None: - date_range_overlaps = date_from <= meas_station_data_model_to - else: - date_range_overlaps = ( - date_from <= meas_station_data_model_to and - date_to_tmp >= meas_station_data_model_from - ) - - if date_range_overlaps: - device_orientation_deg = device_properties.get('device_orientation_deg') - apply_offset_from = (date_from if date_from > meas_station_data_model_from - else meas_station_data_model_from) - if date_to_tmp is None or meas_station_data_model_to is None: - apply_offset_to = date_to_tmp if date_to_tmp is not None else meas_station_data_model_to + logger_offset = wdir_prop.get('logger_measurement_config.offset') + for j, device_properties in enumerate(measurement_station): + meas_station_data_model_from = device_properties.get('date_from') + meas_station_data_model_from = (df.index[0].strftime('%Y-%m-%dT%H:%M:%S') if + meas_station_data_model_from is None or meas_station_data_model_from == + DATE_INSTEAD_OF_NONE else meas_station_data_model_from) + meas_station_data_model_to = device_properties.get('date_to') + # If the last logger properties date to has been explicitly set as the last timestamp of the + # dataset, set it to None. This avoids missing this timestamp due to [date_from, date_to) logic + if meas_station_data_model_to is not None: + if pd.to_datetime(meas_station_data_model_to) >= df.index[-1]: + meas_station_data_model_to = None + # If [date_from, date_to) convention has not been used, we force this convention by setting + # meas_station_data_model_to to the next_meas_station_data_model_from of the next + # measurement_station property. + if j < len(measurement_station_items) - 1: + next_meas_station_data_model_from = measurement_station[j+1].get('date_from') + if next_meas_station_data_model_from != meas_station_data_model_to: + meas_station_data_model_to = next_meas_station_data_model_from + + if date_to is None or date_to == DATE_INSTEAD_OF_NONE: + date_to_tmp = meas_station_data_model_to else: - apply_offset_to = min(date_to_tmp, meas_station_data_model_to) - height = wdir_prop.get('height_m') - - df[name], applied_results = _apply_dir_offset_target_orientation( - df[name], logger_offset, device_orientation_deg, apply_offset_from, apply_offset_to, - target_orientation_name='device orientation', heights=height, - target_orientation_table_name="Device Orientation [deg]" + date_to_tmp = date_to + + date_range_overlaps = False + if meas_station_data_model_to is None and date_to_tmp is None: + date_range_overlaps = True + elif meas_station_data_model_to is None: + date_range_overlaps = date_to_tmp is None or date_to_tmp >= meas_station_data_model_from + elif date_to_tmp is None: + date_range_overlaps = date_from <= meas_station_data_model_to + else: + date_range_overlaps = ( + date_from <= meas_station_data_model_to and + date_to_tmp >= meas_station_data_model_from ) - - rows.append(applied_results) - else: - wdir_not_in_dataset = True - col_not_in_data.append(name) + + if date_range_overlaps: + device_orientation_deg = device_properties.get('device_orientation_deg') + apply_offset_from = (date_from if date_from > meas_station_data_model_from + else meas_station_data_model_from) + if date_to_tmp is None or meas_station_data_model_to is None: + apply_offset_to = date_to_tmp if date_to_tmp is not None else meas_station_data_model_to + else: + apply_offset_to = min(date_to_tmp, meas_station_data_model_to) + height = wdir_prop.get('height_m') + + df[var_name], applied_results = _apply_dir_offset_target_orientation( + df[var_name], logger_offset, device_orientation_deg, apply_offset_from, apply_offset_to, + target_orientation_name='device orientation', heights=height, + target_orientation_table_name="Device Orientation [deg]" + ) + + rows.append(applied_results) + else: + wdir_not_in_dataset = True + col_not_in_data.append(var_name) if wdir_not_in_dataset: indexes = np.unique(col_not_in_data, return_index=True)[1] col_not_in_data = [col_not_in_data[index] for index in sorted(indexes)] print_text = "Following wind direction measurement(s) reported in the 'measurement_station' input " \ - "not found in the data" + "not found in the data" if wdir_cols: print(print_text + ' for the requested `wdir_cols`: {}.'.format(utils.bold(str(col_not_in_data)))) else: print(print_text + ': {}.'.format(utils.bold(str(col_not_in_data)))) if col_not_in_datamodel: - print('No device orientation offset applied to following `wdir_cols` requested measurement(s) ' \ - 'as no wind direction measurement type found in `measurement_station` input for these: {}.' + print('No device orientation offset applied to following `wdir_cols` requested measurement(s) ' + 'as no wind direction measurement type found in `measurement_station` input for these: {}.' .format(utils.bold(str(col_not_in_datamodel)))) # if a Series is sent, send back a Series if isinstance(data, pd.Series): diff --git a/tests/test_transform.py b/tests/test_transform.py index d7acedd6..02adee7e 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -180,6 +180,37 @@ def test_apply_wind_vane_dead_band_offset(): assert (data1.fillna(0).round(10) == data['Dir78mS'].fillna(0).round(10)).all() + + data_edited = copy.deepcopy(DATA) + data_edited['Dir78mS_max'] = data_edited['Dir78mS'] + 10 + + test_dict = STATION.measurements.wdirs + test_dict['Dir78mS'][0]['logger_measurement_config.offset'] = 300 + test_dict['Dir78mS'][0]['logger_measurement_config.column_name'] = [{'column_name': 'Dir78mS', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'}, + {'column_name': 'Dir78mSStd', + 'statistic_type_id': 'sd', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'}, + {'column_name': 'Dir78mSMax', + 'statistic_type_id': 'max', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'}, + {'column_name': 'Dir78mSMin', + 'statistic_type_id': 'min', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'}] + result_data = bw.apply_wind_vane_deadband_offset( + data_edited, test_dict, inplace=False, apply_to_related_statistics=True + ) + + assert np.allclose(result_data['Dir78mS_max'].values[0], result_data['Dir78mS'].values[0] + 10 - 360) def test_apply_wind_vane_dead_band_offset_table_returned(): @@ -192,6 +223,13 @@ def test_apply_wind_vane_dead_band_offset_table_returned(): 'measurement_type_id': 'wind_direction', 'height_m': 78.0, 'logger_measurement_config.slope': 0.07263, 'logger_measurement_config.offset': 130.87, + 'logger_measurement_config.column_name': [ + {'column_name': 'Dir78mS', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'} + ], 'date_from': '2025-02-20T00:10:00', 'date_to': None, 'vane_dead_band_orientation_deg': None}], @@ -200,6 +238,13 @@ def test_apply_wind_vane_dead_band_offset_table_returned(): 'height_m': 78.0, 'logger_measurement_config.slope': 0.07263, 'logger_measurement_config.offset': None, + 'logger_measurement_config.column_name': [ + {'column_name': 'Dir78_test', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'} + ], 'date_from': '2025-02-20T00:10:00', 'date_to': None, 'vane_dead_band_orientation_deg': None}], @@ -208,6 +253,13 @@ def test_apply_wind_vane_dead_band_offset_table_returned(): 'height_m': 58.0, 'logger_measurement_config.slope': 0.07262, 'logger_measurement_config.offset': 127.75, + 'logger_measurement_config.column_name': [ + {'column_name': 'Dir58mS', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'} + ], 'date_from': '2025-02-20T00:10:00', 'date_to': None, 'vane_dead_band_orientation_deg': 127.75}], @@ -216,6 +268,13 @@ def test_apply_wind_vane_dead_band_offset_table_returned(): 'height_m': 58.0, 'logger_measurement_config.slope': 0.07262, 'logger_measurement_config.offset': None, + 'logger_measurement_config.column_name': [ + {'column_name': 'Dir58_test', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'} + ], 'date_from': '2025-02-20T00:10:00', 'date_to': None, 'vane_dead_band_orientation_deg': 127.75}], @@ -224,6 +283,13 @@ def test_apply_wind_vane_dead_band_offset_table_returned(): 'height_m': 38.0, 'logger_measurement_config.slope': 22.5, 'logger_measurement_config.offset': 36.72, + 'logger_measurement_config.column_name': [ + {'column_name': 'Dir38mS', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'} + ], 'date_from': '2025-02-20T00:10:00', 'date_to': None, 'vane_dead_band_orientation_deg': 2}] @@ -275,6 +341,13 @@ def test_apply_wind_vane_dead_band_offset_table_returned(): 'measurement_type_id': 'wind_direction', 'height_m': 78.0, 'logger_measurement_config.slope': 0.07263, 'logger_measurement_config.offset': 132.87, + 'logger_measurement_config.column_name': [ + {'column_name': 'Dir78mS', + 'statistic_type_id': 'avg', + 'is_ignored': False, + 'notes': None, + 'update_at': '2021-02-24T17:04:41'} + ], 'date_from': '2016-10-20T08:10:00', 'date_to': None, 'vane_dead_band_orientation_deg': 3} From a3536ea61224a68863d84e2d30fdee30dd08e2b5 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Thu, 12 Feb 2026 16:29:04 +0000 Subject: [PATCH 33/35] update version to 2.5.0 --- brightwind/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/brightwind/__init__.py b/brightwind/__init__.py index d865d7f5..cc8c2ade 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -34,4 +34,4 @@ __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.4.0' +__version__ = '2.5.0' diff --git a/setup.py b/setup.py index 3c4fe9c7..863b103a 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def get_version(rel_path): }, url='https://github.com/brightwind-dev/brightwind.git', # UPDATE VERSION NUMBER HERE: - download_url='https://github.com/brightwind-dev/brightwind/archive/v2.4.0.tar.gz', + download_url='https://github.com/brightwind-dev/brightwind/archive/v2.5.0.tar.gz', license='MIT', author='Stephen Holleran of BrightWind Ltd', author_email='stephen@brightwindanalysis.com', From 01821f65d0fbf14191190560c60a95af10b4e033 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Thu, 12 Feb 2026 16:35:27 +0000 Subject: [PATCH 34/35] add release dates to changelog --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a30c5e32..5b84f215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.5.0] +12-Feb-2026 ### New Features and Enhancements 1. Updated `calc_air_density()` to incorporate additional methods. The methods differ in their approach to deriving water vapour pressure using `_calc_water_vapour_pressure_Pa()` and include the pre-existing 'IEC' method (default). The two new methods are based on the Herman Wobus approximation (`_calc_water_saturation_vapour_pressure_Pa()`) and require either relative humidity or dew point temperature as inputs. ([#539](https://github.com/brightwind-dev/brightwind/issues/539)). @@ -25,6 +26,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0] +17-Dec-2025 ### New Features and Enhancements 1. Added `scale_air_pressure_to_height()` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height. ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) @@ -53,8 +55,10 @@ Additional labels for pre-release and build metadata are available as extensions 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) 2. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) - +--- ## [2.3.0] +14-Apr-2025 + This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. Key improvements include more reliable wind and solar data handling, expanded plotting capabilities (including colormap support and better legends), and the introduction of new functions for downloading and applying cleaning rules From e06dceb05b01c4b80ac0241dcf02ea57d9048677 Mon Sep 17 00:00:00 2001 From: sararafter <183604246+sararafter@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:06:33 +0000 Subject: [PATCH 35/35] change tests to assert warning rather than error --- brightwind/analyse/analyse.py | 3 ++- tests/test_analyse.py | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 0173ff8b..9736ae90 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -2507,7 +2507,8 @@ def calc_rel_humidity_from_dew_point(dew_point_temperature_degC: Union[float, pd # Calculate relative humidity rel_humidity_percent = 100*(_calc_water_saturation_vapour_pressure_Pa(dew_point_temperature_degC) / _calc_water_saturation_vapour_pressure_Pa(air_temperature_degC)) - rel_humidity_percent.name = 'relative_humidity' + if isinstance(rel_humidity_percent, pd.Series): + rel_humidity_percent.name = 'relative_humidity' return rel_humidity_percent diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 0f3f7862..fa64e632 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -719,16 +719,16 @@ def test_calc_air_density(): " provided.") # Check error is raised when dew_point_temperature_degC > temperature - with pytest.raises(ValueError) as except_info: + msg = ("Detected dew_point_temperature_degC values which are greater than corresponding air temperature " + "values.\n This implies that the relative humidity would be greater than 100%.") + with pytest.warns(UserWarning, match=msg): bw.calc_air_density(1, 935, dew_point_temperature_degC=5, calc_method='HermanWobus_from_dew_point') - assert str(except_info.value) == ("dew_point_temperature_degC cannot be greater than temperature.") - with pytest.raises(ValueError) as except_info: + with pytest.warns(UserWarning, match=msg): bw.calc_air_density(DATA.T2m.tail(3), DATA.P2m.tail(3), dew_point_temperature_degC=DATA.T2m.tail(3) + 5, calc_method='HermanWobus_from_dew_point') - assert str(except_info.value) == ("dew_point_temperature_degC cannot be greater than temperature.") # Test Series inputs dew_point_temp = DATA.T2m - 0.5 @@ -756,21 +756,20 @@ def test_dist_matrix_by_direction_sector(): def test_calc_rel_humidity_from_dew_point(): # test error for dew point greater than temperature - with pytest.raises(ValueError) as except_info: + msg = ("Detected dew_point_temperature_degC values which are greater than corresponding air temperature " + "values.\n This will result in a calculated relative humidity value greater than 100%.") + with pytest.warns(UserWarning, match=msg): bw.calc_rel_humidity_from_dew_point(11, 10) - assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." - with pytest.raises(ValueError) as except_info: + with pytest.warns(UserWarning, match=msg): bw.calc_rel_humidity_from_dew_point(dew_point_temperature_degC=DATA.T2m.tail(3)+[-1, 0, 1], air_temperature_degC=DATA.T2m.tail(3)) - assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." - with pytest.raises(ValueError) as except_info: + with pytest.warns(UserWarning, match=msg): bw.calc_rel_humidity_from_dew_point(dew_point_temperature_degC=20, air_temperature_degC=DATA.T2m.tail(3)) - assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." - with pytest.raises(ValueError) as except_info: + with pytest.warns(UserWarning, match=msg): bw.calc_rel_humidity_from_dew_point(dew_point_temperature_degC=DATA.T2m.tail(3), air_temperature_degC=0) - assert str(except_info.value) == "dew_point_temperature_degC cannot be greater than temperature." + # test error for Series inputs with different lengths with pytest.raises(ValueError) as except_info: bw.calc_rel_humidity_from_dew_point(DATA.T2m.tail(5), DATA.T2m.tail(3))