From c5de546c2185e3f7584da8f511955765c6ac59bf Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Mon, 14 Apr 2025 15:15:55 +0100 Subject: [PATCH 01/27] prep for new developments --- CHANGELOG.md | 2 ++ brightwind/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e51904e..3b6bafec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Additional labels for pre-release and build metadata are available as extensions --- +## [2.4.0-dev] +1. ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 85e5745d..20152832 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -12,4 +12,4 @@ __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.3.0' +__version__ = '2.4.0-dev' From f596c79540f0550992515d7c2d8f2c3f35971ca8 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Wed, 11 Jun 2025 15:12:34 +0200 Subject: [PATCH 02/27] iss #458 numpy pandas update --- brightwind/analyse/analyse.py | 2 +- brightwind/analyse/plot.py | 8 ++--- brightwind/analyse/shear.py | 2 +- brightwind/load/station.py | 4 +-- brightwind/transform/transform.py | 16 ++++----- requirements.txt | 4 +-- tests/test_correlation.py | 60 +++++++++++++++---------------- tests/test_plot.py | 4 +-- tests/test_transform.py | 40 ++++++++++----------- 9 files changed, 70 insertions(+), 70 deletions(-) diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 1ad79b1e..936efc4e 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -740,7 +740,7 @@ def custom_agg(x): graph = plt.figure(figsize=(15, 8)) ax = graph.add_axes([0.1, 0.1, 0.8, 0.8]) - bw_plt._bar_subplot(distributions.replace([np.inf, -np.inf], np.NAN), x_label=x_label, y_label=aggregation_method, + bw_plt._bar_subplot(distributions.replace([np.inf, -np.inf], np.nan), x_label=x_label, y_label=aggregation_method, max_bar_axis_limit=max_y_value, bin_tick_labels=bin_labels, bar_tick_label_format=bar_tick_label_format, legend=legend, total_width=0.8, ax=ax) plt.close() diff --git a/brightwind/analyse/plot.py b/brightwind/analyse/plot.py index 3f18d642..95303b59 100644 --- a/brightwind/analyse/plot.py +++ b/brightwind/analyse/plot.py @@ -1455,7 +1455,7 @@ def _bar_subplot(data, x_label=None, y_label=None, min_bar_axis_limit=None, max_ var_to_bin_against=data['Spd80mN'].to_frame(), aggregation_method = '%frequency') fig = plt.figure(figsize=(15, 8)) - bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), y_label='%frequency') + bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.nan).dropna(), y_label='%frequency') """ @@ -1638,7 +1638,7 @@ def plot_freq_distribution(data, max_y_value=None, x_tick_labels=None, x_label=N distribution = bw.analyse.analyse._derive_distribution(data['Spd40mN'], var_to_bin_against=data['Spd40mN'], bins=None, aggregation_method = '%frequency').rename('Spd40mN') - bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), + bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.nan).dropna(), max_y_value=None,x_tick_labels=[], x_label=None, y_label='%frequency') @@ -1651,7 +1651,7 @@ def plot_freq_distribution(data, max_y_value=None, x_tick_labels=None, x_label=N aggregation_method='count').rename('Spd80mN') bw.analyse.plot.plot_freq_distribution(pd.concat([distribution1, distribution2], axis=1 - ).replace([np.inf, -np.inf], np.NAN).dropna(), + ).replace([np.inf, -np.inf], np.nan).dropna(), max_y_value=None, x_tick_labels=None, x_label=None, y_label='count', total_width=1, legend=True) @@ -1667,7 +1667,7 @@ def plot_freq_distribution(data, max_y_value=None, x_tick_labels=None, x_label=N fig = plt.figure(figsize=(15, 8)) ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) - _bar_subplot(data.replace([np.inf, -np.inf], np.NAN), x_label=x_label, + _bar_subplot(data.replace([np.inf, -np.inf], np.nan), x_label=x_label, y_label=y_label, max_bar_axis_limit=max_y_value, bin_tick_labels=x_tick_labels, bar_tick_label_format=bar_tick_label_format, legend=legend, total_width=total_width, ax=ax) diff --git a/brightwind/analyse/shear.py b/brightwind/analyse/shear.py index 6cefd1d2..592147f9 100644 --- a/brightwind/analyse/shear.py +++ b/brightwind/analyse/shear.py @@ -1096,7 +1096,7 @@ def _fill_df_12x24(data): idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1H') # create new dataframe with 24 rows only interval number of unique values - df = pd.DataFrame({cols: [np.NaN] for cols in df_copy.columns}, index=pd.DatetimeIndex(idx).time) + df = pd.DataFrame({cols: [np.nan] for cols in df_copy.columns}, index=pd.DatetimeIndex(idx).time) df = pd.concat( [(df[df_copy.index[0].hour:]), (df[:df_copy.index[0].hour])], axis=0) diff --git a/brightwind/load/station.py b/brightwind/load/station.py index 3ed0d25f..3a9bd29c 100644 --- a/brightwind/load/station.py +++ b/brightwind/load/station.py @@ -1165,11 +1165,11 @@ def get_heights(self, names=None, measurement_type_id=None): name_found = True break elif meas_point['name'] == name and meas_point.get('height_m') is None: - heights.append(np.NaN) + heights.append(np.nan) name_found = True break if name_found is False: - heights.append(np.NaN) + heights.append(np.nan) return heights diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 1c17e7f3..1d301593 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -240,18 +240,18 @@ def _get_overlapping_data(df1, df2, averaging_prd=None): # averaging will start from this timestamp. if not (df2.index == start).any(): if type(df2) == pd.DataFrame: - df2 = pd.concat([df2, pd.DataFrame({cols: [np.NaN] for cols in df2.columns}, + df2 = pd.concat([df2, pd.DataFrame({cols: [np.nan] for cols in df2.columns}, index=[pd.to_datetime(start)])]) else: - df2[pd.to_datetime(start)] = np.NaN + df2[pd.to_datetime(start)] = np.nan df2.sort_index(inplace=True) if not (df1.index == start).any(): - # df1.loc[pd.to_datetime(start)] = np.NaN + # df1.loc[pd.to_datetime(start)] = np.nan if type(df1) == pd.DataFrame: - df1 = pd.concat([df1, pd.DataFrame({cols: [np.NaN] for cols in df1.columns}, + df1 = pd.concat([df1, pd.DataFrame({cols: [np.nan] for cols in df1.columns}, index=[pd.to_datetime(start)])]) else: - df1[pd.to_datetime(start)] = np.NaN + df1[pd.to_datetime(start)] = np.nan df1.sort_index(inplace=True) return df1[start:], df2[start:] @@ -538,7 +538,7 @@ def _vector_avg_of_wdirs_dataframe(wdirs, wspds=None): # means there is no wind direction => return NaN nan_mask = (avg_dir_df['sine'] == 0) & (avg_dir_df['cosine'] == 0) avg_dir_df['avg_dir'] = np.rad2deg(np.arctan2(sine, cosine)) % 360 - avg_dir_df['avg_dir'][nan_mask] = np.NaN + avg_dir_df['avg_dir'][nan_mask] = np.nan return avg_dir_df['avg_dir'] @@ -582,7 +582,7 @@ def _vector_avg_of_wdirs_list(wdirs, wspds=None): wspds = a[1] # if the resulting wdir array is empty, return NAN if wdirs.size == 0: - return np.NaN + return np.nan if wspds is None: sine = np.mean(np.round(np.sin(np.deg2rad(wdirs)), 5)) # sin of each angle, East component @@ -594,7 +594,7 @@ def _vector_avg_of_wdirs_list(wdirs, wspds=None): # If both sine and cosine result in zero then all the directions cancel and you end up where you started which # means there is no wind direction => return NaN if sine == 0 and cosine == 0: - avg_dir = np.NaN + avg_dir = np.nan else: avg_dir = np.rad2deg(np.arctan2(sine, cosine)) % 360 if avg_dir == 360.0: # preference to have 0 returned instead of 360 diff --git a/requirements.txt b/requirements.txt index 337e20f7..8be85f60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pandas >= 0.24.0, <=2.0.1 -numpy >= 1.16.4, <2.0.0 +pandas >= 0.24.0 +numpy >= 1.16.4 scikit-learn >= 0.19.1 matplotlib >= 3.0.3 requests >= 2.20.0 diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 9e247592..62964252 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -5,15 +5,15 @@ import warnings wndspd = 8 -wndspd_df = pd.DataFrame([2, 13, np.NaN, 5, 8]) -wndspd_series = pd.Series([2, 13, np.NaN, 5, 8]) +wndspd_df = pd.DataFrame([2, 13, np.nan, 5, 8]) +wndspd_series = pd.Series([2, 13, np.nan, 5, 8]) current_slope = 0.045 current_offset = 0.235 new_slope = 0.046 new_offset = 0.236 wndspd_adj = 8.173555555555556 -wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) -wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) +wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) +wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) DATA = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) DATA_CLND = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) @@ -209,62 +209,62 @@ def test_synthesize(): '2017-06-05 00:00:00': 9.88184374999999, '2017-06-15 00:00:00': 7.639711111111101, '2017-06-25 00:00:00': 9.144633101851865, - '2017-07-05 00:00:00': np.NaN, - '2017-07-15 00:00:00': np.NaN, + '2017-07-05 00:00:00': np.nan, + '2017-07-15 00:00:00': np.nan, '2017-07-25 00:00:00': 7.127115740740741, '2017-08-04 00:00:00': 6.4158555555555505, '2017-08-14 00:00:00': 7.510521527777763, '2017-08-24 00:00:00': 5.943415798611104, - '2017-09-03 00:00:00': np.NaN, - '2017-09-13 00:00:00': np.NaN, + '2017-09-03 00:00:00': np.nan, + '2017-09-13 00:00:00': np.nan, '2017-09-23 00:00:00': 12.601222222222226, '2017-10-03 00:00:00': 9.425352777777784, '2017-10-13 00:00:00': 9.466307638888878, '2017-10-23 00:00:00': 8.84323971518988}} - result_ord_lst_sq_dir = {'Spd80mN_Synthesized': {'2016-03-01': np.NaN, + result_ord_lst_sq_dir = {'Spd80mN_Synthesized': {'2016-03-01': np.nan, '2016-04-01': 6.598875, - '2016-05-01': np.NaN, + '2016-05-01': np.nan, '2016-06-01': 5.108156, '2016-07-01': 6.319782, '2016-08-01': 7.093956, - '2016-09-01': np.NaN, + '2016-09-01': np.nan, '2016-10-01': 6.669446, - '2016-11-01': np.NaN, + '2016-11-01': np.nan, '2016-12-01': 8.900778, '2017-01-01': 9.501281, '2017-02-01': 9.134509, - '2017-03-01': np.NaN, + '2017-03-01': np.nan, '2017-04-01': 7.783390, - '2017-05-01': np.NaN, + '2017-05-01': np.nan, '2017-06-01': 8.525249, '2017-08-01': 6.715885, '2017-10-01': 9.479016}} - result_speed_sort = {'Spd80mN_Synthesized': {'2016-03-01': np.NaN, + result_speed_sort = {'Spd80mN_Synthesized': {'2016-03-01': np.nan, '2016-04-01': 6.598875, - '2016-05-01': np.NaN, + '2016-05-01': np.nan, '2016-06-01': 5.108156, - '2016-07-01': np.NaN, + '2016-07-01': np.nan, '2016-08-01': 7.093956, - '2016-09-01': np.NaN, + '2016-09-01': np.nan, '2016-10-01': 6.669446, - '2016-11-01': np.NaN, + '2016-11-01': np.nan, '2016-12-01': 8.900778, - '2017-01-01': np.NaN, + '2017-01-01': np.nan, '2017-02-01': 9.134509, - '2017-03-01': np.NaN}, - 'Dir78mS_Synthesized': {'2016-03-01': np.NaN, + '2017-03-01': np.nan}, + 'Dir78mS_Synthesized': {'2016-03-01': np.nan, '2016-04-01': 318.639591, - '2016-05-01': np.NaN, + '2016-05-01': np.nan, '2016-06-01': 129.720897, - '2016-07-01': np.NaN, + '2016-07-01': np.nan, '2016-08-01': 235.377543, '2016-09-01': 223.965222, '2016-10-01': 114.451746, - '2016-11-01': np.NaN, + '2016-11-01': np.nan, '2016-12-01': 219.815444, '2017-01-01': 237.667123, '2017-02-01': 197.538174, - '2017-03-01': np.NaN}} + '2017-03-01': np.nan}} data_spd80mn_even_months = DATA_CLND['Spd80mN'][DATA_CLND.index.month.isin([2, 4, 6, 8, 10, 12])] # Test the synthesise for when the target data starts before the reference data. @@ -305,10 +305,10 @@ def test_synthesize(): # Test the synthesise when SpeedSort correlation is used using 10 min averaging period. data_test = DATA_CLND[['Spd80mN', 'Spd60mN', 'Dir78mS', 'Dir58mS']].copy() - data_test['Dir78mS']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.NaN - data_test['Spd80mN']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.NaN - data_test['Dir58mS']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.NaN - data_test['Spd60mN']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.NaN + data_test['Dir78mS']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan + data_test['Spd80mN']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan + data_test['Dir58mS']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan + data_test['Spd60mN']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan ss_cor = bw.Correl.SpeedSort(data_test['Spd80mN'], data_test['Dir78mS'], data_test['Spd60mN'], data_test['Dir58mS'], averaging_prd='10min') ss_cor.run() diff --git a/tests/test_plot.py b/tests/test_plot.py index 68bf839b..235f935c 100644 --- a/tests/test_plot.py +++ b/tests/test_plot.py @@ -184,7 +184,7 @@ def test_bar_subplot(): var_to_bin_against=DATA['Spd80mN'].to_frame(), aggregation_method='%frequency') fig = plt.figure(figsize=(15, 8)) - bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), y_label='%frequency') + bw.analyse.plot._bar_subplot(distribution.replace([np.inf, -np.inf], np.nan).dropna(), y_label='%frequency') assert True @@ -194,7 +194,7 @@ def test_plot_freq_distribution(): distribution = bw.analyse.analyse._derive_distribution(DATA['Spd40mN'], var_to_bin_against=DATA['Spd40mN'], bins=None, aggregation_method='%frequency').rename('Spd40mN') - bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.NAN).dropna(), + bw.analyse.plot.plot_freq_distribution(distribution.replace([np.inf, -np.inf], np.nan).dropna(), max_y_value=None, x_tick_labels=[], x_label=None, y_label='%frequency') diff --git a/tests/test_transform.py b/tests/test_transform.py index d26d916d..7e3f176d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -14,15 +14,15 @@ wndspd = 8 -wndspd_df = pd.DataFrame([2, 13, np.NaN, 5, 8]) -wndspd_series = pd.Series([2, 13, np.NaN, 5, 8]) +wndspd_df = pd.DataFrame([2, 13, np.nan, 5, 8]) +wndspd_series = pd.Series([2, 13, np.nan, 5, 8]) current_slope = 0.045 current_offset = 0.235 new_slope = 0.046 new_offset = 0.236 wndspd_adj = 8.173555555555556 -wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) -wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.NaN, 5.106888888888888, 8.173555555555556]) +wndspd_adj_df = pd.DataFrame([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) +wndspd_adj_series = pd.Series([2.0402222222222224, 13.284666666666668, np.nan, 5.106888888888888, 8.173555555555556]) ref_date = pd.to_datetime('2000-01-01') DATA = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) @@ -50,13 +50,13 @@ def test_selective_avg(): days = pd.date_range(date_today, date_today + datetime.timedelta(24), freq='D') data = pd.DataFrame({'DTM': days}) data = data.set_index('DTM') - data['Spd1'] = [1, np.NaN, 1, 1, 1, 1, 1, 1, 1, np.NaN, 1, 1, 1, 1, np.NaN, 1, 1, np.NaN, 1, 1, 1, 1, np.NaN, 1, 1] - data['Spd2'] = [2, 2, np.NaN, 2, 2, 2, 2, 2, np.NaN, 2, 2, 2, 2, np.NaN, 2, 2, 2, np.NaN, 2, 2, 2, 2, 2, np.NaN, 2] - data['Dir'] = [0, 15, 30, 45, np.NaN, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, - 315, np.NaN, 345, 360] + data['Spd1'] = [1, np.nan, 1, 1, 1, 1, 1, 1, 1, np.nan, 1, 1, 1, 1, np.nan, 1, 1, np.nan, 1, 1, 1, 1, np.nan, 1, 1] + data['Spd2'] = [2, 2, np.nan, 2, 2, 2, 2, 2, np.nan, 2, 2, 2, 2, np.nan, 2, 2, 2, np.nan, 2, 2, 2, 2, 2, np.nan, 2] + data['Dir'] = [0, 15, 30, 45, np.nan, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, + 315, np.nan, 345, 360] # Test Case 1: Neither boom is near 0-360 crossover - result = np.array([1.5, 2, 1, 1.5, 1.5, 1.5, 1.5, 2, 1, 2, 2, 2, 1.5, 1, 2, 1.5, 1.5, np.NaN, + result = np.array([1.5, 2, 1, 1.5, 1.5, 1.5, 1.5, 2, 1, 2, 2, 2, 1.5, 1, 2, 1.5, 1.5, np.nan, 1.5, 1, 1, 1, 2, 1, 1.5]) bw.selective_avg(data[['Spd1']], data[['Spd2']], data[['Dir']], boom_dir_1=315, boom_dir_2=135, sector_width=60) @@ -65,21 +65,21 @@ def test_selective_avg(): assert np_array_equal(sel_avg, result) # Test Case 2: Boom 1 is near 0-360 crossover - result = np.array([1.0, 2.0, 1.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, 2.0, 1.0, 2.0, 2.0, 1.5, np.NaN, + result = np.array([1.0, 2.0, 1.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, 2.0, 1.0, 2.0, 2.0, 1.5, np.nan, 1.5, 1.5, 1.5, 1.5, 2.0, 1.0, 1.0]) sel_avg = np.array(bw.selective_avg(data.Spd1, data.Spd2, data.Dir, boom_dir_1=20, boom_dir_2=200, sector_width=60)) assert np_array_equal(sel_avg, result) # Test Case 3: Boom 2 is near 0-360 crossover - result = np.array([2.0, 2.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.5, 1.5, np.NaN, + result = np.array([2.0, 2.0, 1.0, 1.5, 1.5, 1.5, 1.5, 1.5, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.5, 1.5, np.nan, 1.5, 1.5, 1.5, 1.5, 2.0, 1.0, 2.0]) sel_avg = np.array(bw.selective_avg(data.Spd1, data.Spd2, data.Dir, boom_dir_1=175, boom_dir_2=355, sector_width=60)) assert np_array_equal(sel_avg, result) # Test Case 4: Booms at 90 deg to each other - result = np.array([1.0, 2.0, 1.0, 1.5, 1.5, 2.0, 2.0, 2.0, 1.0, 2.0, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, np.NaN, + result = np.array([1.0, 2.0, 1.0, 1.5, 1.5, 2.0, 2.0, 2.0, 1.0, 2.0, 1.5, 1.5, 1.5, 1.0, 2.0, 1.5, 1.5, np.nan, 1.5, 1.5, 1.5, 1.5, 2.0, 1.0, 1.0]) sel_avg = np.array(bw.selective_avg(data.Spd1, data.Spd2, data.Dir, boom_dir_1=270, boom_dir_2=180, sector_width=60)) @@ -161,13 +161,13 @@ def test_offset_wind_direction_float(): def test_offset_wind_direction_df(): - wdir_df_offset = pd.DataFrame([355, 15, np.NaN, 25, 335]) - assert wdir_df_offset.equals(bw.offset_wind_direction(pd.DataFrame([10, 30, np.NaN, 40, 350]), 345)) + wdir_df_offset = pd.DataFrame([355, 15, np.nan, 25, 335]) + assert wdir_df_offset.equals(bw.offset_wind_direction(pd.DataFrame([10, 30, np.nan, 40, 350]), 345)) def test_offset_wind_direction_series(): - wdir_series_offset = pd.Series([355, 15, np.NaN, 25, 335]) - assert wdir_series_offset.equals(bw.offset_wind_direction(pd.Series([10, 30, np.NaN, 40, 350]), 345)) + wdir_series_offset = pd.Series([355, 15, np.nan, 25, 335]) + assert wdir_series_offset.equals(bw.offset_wind_direction(pd.Series([10, 30, np.nan, 40, 350]), 345)) def test_apply_wind_vane_dead_band_offset(): @@ -443,10 +443,10 @@ def test_average_wdirs(): assert bw.average_wdirs(wdirs) == 0.0 wdirs = np.array([0, 180]) - assert bw.average_wdirs(wdirs) is np.NaN + assert bw.average_wdirs(wdirs) is np.nan wdirs = np.array([90, 270]) - assert bw.average_wdirs(wdirs) is np.NaN + assert bw.average_wdirs(wdirs) is np.nan wdirs = np.array([45, 135]) assert bw.average_wdirs(wdirs) == 90 @@ -455,7 +455,7 @@ def test_average_wdirs(): assert bw.average_wdirs(wdirs) == 180 wdirs = np.array([45, 315, 225, 135]) - assert bw.average_wdirs(wdirs) is np.NaN + assert bw.average_wdirs(wdirs) is np.nan wdirs = np.array([225, 315]) assert bw.average_wdirs(wdirs) == 270 @@ -473,7 +473,7 @@ def test_average_wdirs(): assert round(bw.average_wdirs(wdirs_with_nan, wspds_with_nan), 3) == 15.0 wspds_with_nan = [np.nan, np.nan, np.nan] - assert bw.average_wdirs(wdirs_with_nan, wspds_with_nan) is np.NaN + assert bw.average_wdirs(wdirs_with_nan, wspds_with_nan) is np.nan wspds_with_nan = [3, 4, np.nan] assert round(bw.average_wdirs(pd.Series(wdirs_with_nan), pd.Series(wspds_with_nan)), 3) == 15.0 From 453b62326ad643645667dc9283f2d353384a3c58 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:12:37 +0200 Subject: [PATCH 03/27] loosen pandas numpy requirements in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bad46d06..603b3e52 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ def get_version(rel_path): keywords=['BRIGHT', 'WIND', 'RESOURCE', 'DATA', 'ANALYSTS', 'PROCESSING', 'WASP', 'ROSE', 'WINDFARMER', 'OPENWIND', 'WIND PRO', 'WINDOGRAPHER'], install_requires=[ - 'pandas>=0.24.0, <=2.0.1', - 'numpy>=1.16.4, <2.0.0', + 'pandas>=0.24.0', + 'numpy>=1.16.4', 'scikit-learn>=0.19.1', 'matplotlib>=3.0.3', 'requests>=2.20.0', From b7c793a4f2bc083a40550d496816178f6b90bce5 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:53:58 +0200 Subject: [PATCH 04/27] add jinja2 to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8be85f60..e6c8147e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ ipython >= 7.4.0 gmaps >= 0.9.0 colormap >= 1.0.1 easydev >= 0.10.0 -jsonschema >= 4.17.3 \ No newline at end of file +jsonschema >= 4.17.3 +jinja2 >= 3.0.0 \ No newline at end of file From d314ed8947c88644305b9f4de6940bcbee083413 Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Fri, 13 Jun 2025 13:01:04 +0200 Subject: [PATCH 05/27] iss #437 add env to test.yml --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1c512661..2fda0033 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,10 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +env: + BRIGHTHUB_EMAIL: email + BRIGHTHUB_PASSWORD: password + jobs: build: From 3b871ef982db2f01adfa53fd62f4548de4659a1c Mon Sep 17 00:00:00 2001 From: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:07:31 +0200 Subject: [PATCH 06/27] add github secrets --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2fda0033..c6511c45 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,8 +10,8 @@ on: workflow_dispatch: env: - BRIGHTHUB_EMAIL: email - BRIGHTHUB_PASSWORD: password + BRIGHTHUB_EMAIL: ${{ secrets.BRIGHTHUB_EMAIL }} + BRIGHTHUB_PASSWORD: ${{ secrets.BRIGHTHUB_PASSWORD }} jobs: build: From b4c62386e73cf54123b6d25eea56ab59babb0f9e Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 11:51:23 +0100 Subject: [PATCH 07/27] iss #458 and #407 fixed bug with pandas v2.0 --- brightwind/load/load.py | 2 +- tests/test_load.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/brightwind/load/load.py b/brightwind/load/load.py index d36f4206..32e344fd 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -88,7 +88,7 @@ def _assemble_df_from_folder(source_folder, file_type, function_to_get_df, print assembled_df = pd.DataFrame() for file_name in files_list: df = function_to_get_df(file_name, **kwargs) - assembled_df = assembled_df.append(df, verify_integrity=True) + assembled_df = pd.concat([assembled_df, df], verify_integrity=True) if print_progress: print("{0} file read and appended".format(file_name)) ctr = ctr + 1 diff --git a/tests/test_load.py b/tests/test_load.py index 0cf480fd..7332d240 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -4,6 +4,7 @@ import pandas as pd import numpy as np import json +from unittest.mock import patch DEMO_DATA_FOLDER = os.path.join(os.path.dirname(__file__), '../brightwind/demo_datasets') @@ -123,6 +124,19 @@ def test_load_csv(): data3['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999)).all().all() assert (data['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999) == data4['2016-01-09 15:30:00':'2016-01-10 23:50:00'].fillna(-999)).all().all() + + # test loading files from folder + bw.export_csv(data[:'2016-01-09 17:00'], os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv')) + bw.export_csv(data['2016-01-09 17:10':'2016-01-09 18:00'], os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')) + with patch('brightwind.load.load._list_files', return_value=[ + os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv'), + os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')]): + assert isinstance(bw.load_csv(DEMO_DATA_FOLDER, '.csv'), pd.DataFrame) + # Remove the temp files + if os.path.exists(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv')): + os.remove(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_first_chunk.csv')) + if os.path.exists(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')): + os.remove(os.path.join(DEMO_DATA_FOLDER, 'temp_test_data_second_chunk.csv')) def test_load_windographer_txt(): From 128383c05528de84ce990157936bfaa569f35d3e Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 12:12:37 +0100 Subject: [PATCH 08/27] iss #458 minor formatting change --- brightwind/load/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 32e344fd..01f85d76 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -2442,7 +2442,7 @@ def apply_cleaning_rules(data, cleaning_rules_file_or_list, inplace=False, repla if utils.is_file_extension(cleaning_rules_file_or_list, ".json"): with open(cleaning_rules_file_or_list) as file: cleaning_json = json.load(file) - elif isinstance(cleaning_rules_file_or_list, List): + elif isinstance(cleaning_rules_file_or_list, list): if not all(isinstance(item, dict) for item in cleaning_rules_file_or_list): raise TypeError("All elements in the `cleaning_rules_file_or_list` must be dictionaries.") cleaning_json = cleaning_rules_file_or_list From 0e78ef5c69a47cea52cd0cfef496e2117552c4ed Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 12:55:46 +0100 Subject: [PATCH 09/27] iss #458 updated max requirements for pandas and python --- requirements.txt | 2 +- setup.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e6c8147e..2d0f8a09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pandas >= 0.24.0 +pandas >=0.24.0, <3.0.0, numpy >= 1.16.4 scikit-learn >= 0.19.1 matplotlib >= 3.0.3 diff --git a/setup.py b/setup.py index 603b3e52..595c573c 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ def get_version(rel_path): keywords=['BRIGHT', 'WIND', 'RESOURCE', 'DATA', 'ANALYSTS', 'PROCESSING', 'WASP', 'ROSE', 'WINDFARMER', 'OPENWIND', 'WIND PRO', 'WINDOGRAPHER'], install_requires=[ - 'pandas>=0.24.0', + 'pandas>=0.24.0, <3.0.0', 'numpy>=1.16.4', 'scikit-learn>=0.19.1', 'matplotlib>=3.0.3', @@ -67,6 +67,8 @@ def get_version(rel_path): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: MIT License", ], ) From b90f6f15479d9c3e529fe2b3bb0d137fd202ab29 Mon Sep 17 00:00:00 2001 From: BiancaMorandi Date: Wed, 9 Jul 2025 13:11:05 +0100 Subject: [PATCH 10/27] iss #458 updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b6bafec..efbc6df5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0-dev] -1. +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. From 69610ad80020ec56fa975c786e2a522974c67cb8 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Thu, 17 Jul 2025 15:22:58 +0100 Subject: [PATCH 11/27] remove python 3.7 as no longer supported by ubuntu 24.04 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c6511c45..165ac99d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From dac6713e6e0d7fb5ad97ee45ffe3138ccd216716 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:29:01 +0100 Subject: [PATCH 12/27] [Iss531] add function to extrapolate pressure (SR8BM2.5) (#537) * iss #531 added lapse_prs function * iss #531 updated docstring of lapse_prs * iss #531 updated function and argument names and docstring * iss #531 added tests * iss #531 updated changelog * iss #531 updated function name to extrapolate_pressure * iss #531 updated function name to extrapolate_pressure in changelog * iss #531 addressed comments, renamed to scale_pressure_to_height * iss #531 formatting fixes * iss #531 updated names to include air pressure/air temp * iss #531 moved constants to constants.py file * iss #531 updated function to include type declaration * iss #531 removed constants.py and re-added capitalised constants to analyse.py * iss #531 updated formatting and few rewording * iss #531 add error for type and input length and test * iss #531 minor format change * iss #531 update doc string * iss #531 format updates to changelog --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 7 +++ brightwind/analyse/analyse.py | 96 ++++++++++++++++++++++++++++++++++- tests/test_analyse.py | 38 ++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efbc6df5..de7677f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.4.0-dev] 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +### Bug Fixes +1. + +### New Features and Enhancements +1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) + + ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. Key improvements include more reliable wind and solar data handling, expanded plotting capabilities (including diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 936efc4e..31be51cb 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -11,6 +11,7 @@ import warnings import textwrap from matplotlib.ticker import PercentFormatter +from typing import Union __all__ = ['monthly_means', 'momm', @@ -27,7 +28,17 @@ 'basic_stats', 'TI', 'sector_ratio', - 'calc_air_density'] + 'calc_air_density', + 'scale_air_pressure_to_height'] + +# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere +ACCEL_DUE_TO_GRAVITY = 9.80665 + +# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere +TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + +# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere +GAS_CONST_DRY_AIR = 287.05 def dist_matrix(var_series, x_series, y_series, @@ -2065,3 +2076,86 @@ def calc_air_density(temperature, pressure, elevation_ref=None, elevation_site=N raise TypeError('elevation_ref should be a number') else: return ref_air_density + + +def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], + ref_air_temp_degC: Union[float, pd.Series], + ref_height_m: Union[float, int], + target_height_m: Union[float, int]) -> Union[float, int, pd.Series]: + """ + Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) + and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). + + Calculation based on ISO:2533-1975 Standard Atmosphere (https://www.iso.org/obp/ui/#iso:std:iso:2533:en) + as suggested by IEC 61400-12-1: + + scaled_air_pressure = ref_air_pressure_hPa*((1 + (L/ref_air_temp_K)*(target_height_m - ref_height_m))**(-g/(L*R))) + where: + g = 9.80665 is acceleration due to gravity (m/s^2) + L = -0.0065 is the temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = 287.05 is the specific gas constant for dry air (J/K/kg or m2/K/s2) + + :param ref_air_pressure_hPa: Reference air pressure value(s) in hPa (1mbar = 1hPa = 100Pa). + :type ref_air_pressure_hPa: float or int or pandas.Series + :param ref_air_temp_degC: Reference air temperature value(s) in degrees celsius. + :type ref_air_temp_degC: float or int or pandas.Series + :param ref_height_m: Height (in metres) of reference air temperature (ref_air_temp_degC) + and air pressure (ref_air_pressure_hPa). + :type ref_height_m: float or int + :param target_height_m: Height (in metres) which ref_air_pressure_hPa is scaled to. + :type target_height_m float or int + :return: Air pressure at specified height of target_height_m in hPa (1mbar = 1hPa = 100Pa). + Type depends on type(ref_air_pressure_hPa) and type(ref_air_temp_degC) inputs. + :rtype: float or int or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale float value of air pressure + round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, ref_height_m=10, + target_height_m=200), 2) + # 977.45 + + # scale air pressure based on input series of reference air pressure and air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_pressure_to_height(ref_air_pressure_hPa=data['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=data['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10).round(2) + # Timestamp + # 2016-01-09 17:10:00 933.07 + # 2016-01-09 17:20:00 933.07 + # 2016-01-09 17:30:00 933.07 + # 2016-01-09 17:40:00 932.07 + # 2016-01-09 17:50:00 932.07 + # 2016-01-09 18:00:00 932.07 + # dtype: float64 + + """ + + # check input types + for var, var_name in zip([ref_air_pressure_hPa, ref_air_temp_degC], ['ref_air_pressure_hPa', 'ref_air_temp_degC']): + if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series)): + raise TypeError(f"{var_name} must be a float or int or pandas.Series.") + for var, var_name in zip([ref_height_m, target_height_m], ['ref_height_m', 'target_height_m']): + if not (isinstance(var, float) or isinstance(var, int)): + raise TypeError(f"{var_name} must be a float or int.") + + # check dimensions of ref_air_pressure_hPa and ref_air_temp_degC if Series + if isinstance(ref_air_pressure_hPa, pd.Series) and (isinstance(ref_air_temp_degC, pd.Series)): + if len(ref_air_pressure_hPa) != len(ref_air_temp_degC): + raise ValueError("ref_air_pressure_hPa and ref_air_temp_degC must have the same dimensions.") + + # Constants as outlined in ISO:2533 + g = ACCEL_DUE_TO_GRAVITY # Acceleration due to gravity (m/s^2) + L = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE # Temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = GAS_CONST_DRY_AIR # Specific gas constant dry air + + ref_air_temp_K = ref_air_temp_degC + 273.15 # Convert temp units to K + + scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) + ) ** (-g / (L * R))) + + return scaled_air_pressure_hPa diff --git a/tests/test_analyse.py b/tests/test_analyse.py index e6a5bb9d..c1857de5 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -659,3 +659,41 @@ def test_dist_matrix_by_direction_sector(): var_to_bin_by_array=[-8, -5, 5, 10, 15, 20, 26], sectors=8) assert True + +def test_scale_air_pressure_to_height(): + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=200), 2)== 977.45 + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=15), 2) == 999.4 + pd.testing.assert_series_equal(bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=DATA['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=DATA['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data = [933.07, 933.07, 933.07, 932.07, 932.07, 932.07], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + # test error raising for invalid input types + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa='invalid_type', + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m=10) + + # test error raising for invalid slope type + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=1000, + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m='invalid_type') + + # test error raising for mismatched dimensions with Series + with pytest.raises(ValueError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=pd.Series([1, 2, 3]), + ref_air_temp_degC=pd.Series([1, 2]), + ref_height_m=2, + target_height_m=10) From ea976845805b5f6f5b32f9b207eca9fc6e821a0a Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:32:38 +0100 Subject: [PATCH 13/27] [Iss530] Add functions to extrapolate temperature and air density (SR14BM9) (#536) * iss #530 added lapse_temp function * iss #530 updated tests and docstring * iss #530 updated docstring * iss #530 minor docstring update * iss #531 added tests and updated function name to lapse_temperature * Revert "iss #531 added tests and updated function name to lapse_temperature" This reverts commit d41ec28e0b4f7b9e1ecf6f2ff85b8312cbe196e4. * iss #530 updated function name to lapse_temperature * iss #530 added tests * iss #534 added lapse_density function * iss #534 added tests * iss #530 updated changelog * iss #534 updated changelog * iss #530 added private function for lapsing, bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 updated name to extrapolate_temperature and updated to use bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 changed lapse_density to extrapolate_density * iss #530 addressed PR comments, updated function names * iss #530 small formatting fixes * iss #530 updated reference to ref for consistency with other PR * iss #530 formatting and docstring and naming fixes * iss #530 updated functions to include type for each argument * iss 530 changed linear funct to public and fixed several typos and example issues * iss 530 fixed tests for utils and analyse * iss 530 fixed typo changelog * iss 530 updated linear_transform to raise errors and added tests * iss 530 added more tests * iss #530 added source of air density lapse rate to docstring * iss #530 include reference for air density lapse rate in docstring * iss #530 updated input to call global variable * iss #530 update docstring of scale_air_density * iss #530 fix PEP 8 warnings * iss #530 update scale_air_temp docstring * iss #530 updates to linear_transform * iss #530 fix typo, update eg and update docstring * Update CHANGELOG.md --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 4 +- brightwind/analyse/analyse.py | 131 +++++++++++++++++++++++++++++++++- brightwind/utils/utils.py | 89 ++++++++++++++++++++++- tests/test_analyse.py | 45 ++++++++++++ tests/test_utils.py | 56 +++++++++++++++ 5 files changed, 322 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de7677f1..54ec8e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ Additional labels for pre-release and build metadata are available as extensions 1. ### New Features and Enhancements -1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) +1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) +1. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([[#534](https://github.com/brightwind-dev/brightwind/issues/534)]) +1. Added `scale_air_temperature_to_height` to output a air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([[#530](https://github.com/brightwind-dev/brightwind/issues/530)]) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 31be51cb..3dbd5fb0 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -29,7 +29,9 @@ 'TI', 'sector_ratio', 'calc_air_density', - 'scale_air_pressure_to_height'] + 'scale_air_pressure_to_height', + 'scale_air_density_to_height', + 'scale_air_temperature_to_height'] # Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere ACCEL_DUE_TO_GRAVITY = 9.80665 @@ -2159,3 +2161,130 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], ) ** (-g / (L * R))) return scaled_air_pressure_hPa + + +def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_kg_m3_m: float = -0.000113) -> Union[float, pd.Series]: + """ + Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height + (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. + + :param ref_air_density_kg_m3: Reference air density value(s) in kg/m3. + :type ref_air_density_kg_m3: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_density_kg_m3. + :type ref_height_m float + :param target_height_m: Height (in metres) that ref_air_density_kg_m3 is scaled to. + :type target_height_m: float + :param lapse_rate_kg_m3_m: Lapse rate describes how air density changes with increasing height above the + earth's surface in kg/m3/m. + Default value of -0.113 kg/m3 per km above earth's surface (-0.000113 kg/m3/m) + taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014). + :type lapse_rate_kg_m3_m: float + :return: Air density at specified height of target_height_m in kg/m3. Type depends on + type(ref_air_density_kg_m3) input. + :rtype: float or pandas.Series + + **Example usage** + :: + import brightwind as bw + + # scale float value of air density using default lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) + # 1.22174 + + # scale float value of air density using non-default value for lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) + # 1.22 + + # derive air density and scale based on series input values for reference air density + data = bw.load_csv(bw.demo_datasets.demo_data) + test_density = bw.calc_air_density(data.T2m, data.P2m) + bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10) + + Timestamp + 2016-01-09 17:10:00 1.186780 + 2016-01-09 17:20:00 1.187175 + 2016-01-09 17:30:00 1.187747 + 2016-01-09 17:40:00 1.185950 + 2016-01-09 17:50:00 1.186301 + 2016-01-09 18:00:00 1.185686 + dtype: float64 + """ + + scaled_air_density = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) + + return scaled_air_density + + +def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_deg_m: float = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE + ) -> Union[float, pd.Series]: + """ + Linearly scales reference air temperature measurement (ref_air_temperature) from its measurement height + (ref_height_m) to the height specified as target_height_m, by applying the constant lapse_rate_deg_m. + + :param ref_air_temperature: Air temperature value(s) in degrees [for example in Celsius or Kelvin]. + :type ref_air_temperature: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_temperature. + :type ref_height_m: float + :param target_height_m: Height (in metres) that ref_air_temperature is scaled to. + :type target_height_m: float + :param lapse_rate_deg_m: Lapse rate describes how air temperature changes with increasing height + above the earth's surface. + Units should be degrees of temperature per unit of height, e.g. °C/m or K/m. + Default value of -6.5 degrees Celsius per km above the earth's surface + (or -0.0065 °C/m) is commonly used as an approximation of the + atmospheric lapse rate. + In particular, the IEC standards rely on the ISO2533:1975 Standard Atmosphere + which states that a lapse rate of 6.5 K/km is valid for geopotential altitudes + of up to 11 km above earth's surface. + This value was also adopted in WASP 11: + Mortensen, N. G., Heathfield, D. N., Rathmann, O., & Nielsen, M. (2014). + Wind Atlas Analysis and Application Program: WAsP 11 Help Facility. Computer + programme, Department of Wind Energy, Technical University of Denmark + https://orbit.dtu.dk/en/publications/wind-atlas-analysis-and-application-program-wasp-11-help-facility + :type lapse_rate_deg_m: float + :return: Air temperature at specified height of target_height_m in same unit as input + ref_air_temperature [for example in Celsius or Kelvin]. Output type depends on + type(ref_air_temperature) input. + :rtype: float or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale air temperature based on float input value for reference air temperature + bw.scale_air_temperature_to_height(ref_air_temperature=10.0065, ref_height_m=10, target_height_m=11) + # 10.0 + + # scale air temperature based on float input value for reference air temperature with non-default lapse_rate_deg_m + bw.scale_air_temperature_to_height(ref_air_temperature=10, ref_height_m=12, target_height_m=10, + lapse_rate_deg_m=-0.001) + # 10.002 + + # scale air temperature based on series input values for reference air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_temperature_to_height(ref_air_temperature=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=20) + # Timestamp + # 2016-01-09 17:10:00 0.837 + # 2016-01-09 17:20:00 0.746 + # 2016-01-09 17:30:00 0.614 + # 2016-01-09 17:40:00 0.735 + # 2016-01-09 17:50:00 0.654 + # 2016-01-09 18:00:00 0.796 + # Name: T2m, dtype: float64 + """ + + scaled_air_temp = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_temperature, slope=lapse_rate_deg_m) + return scaled_air_temp diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index 410944e4..bd6d6361 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -3,12 +3,14 @@ import os import json from jsonschema import Draft7Validator +from typing import Union __all__ = ['slice_data', 'validate_coverage_threshold', 'is_file', 'is_file_extension', - 'validate_json'] + 'validate_json', + 'linear_transform'] def _range_0_to_360(direction): @@ -252,3 +254,88 @@ def validate_json(json_to_check, schema): print(f"Failed schema part: {error.get('schema_path')}\n") return data_is_valid + + +def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], + x_ref: Union[float, int, np.ndarray, pd.Series], + y_ref: Union[float, int, np.ndarray, pd.Series], + slope: Union[float, int]) -> Union[float, int, np.ndarray, pd.Series]: + """ + Perform a linear transform of known (x_ref, y_ref) to calculate y_target, + using a constant slope and known value(s) of x_target. + + Function applies a linear transformation based on the equation of a straight line with constant slope: + y_target = slope * (x_target - x_ref) + y_ref, + where (x_ref, y_ref) is effectively considered a known point on a line with the inputted slope. + + Note that if pd.Series or np.ndarray inputs are provided for x_target, x_ref and y_ref, + then the transform is performed on a point by point basis, as in, the n^{th} value of + the resulting y_target would be: + y_target[n] = slope * (x_target[n] - x_ref[n]) + y_ref[n], + where (x_ref[n], y_ref[n]) is effectively considered a known point on a line with the inputted slope. + Therefore all three of x_target, x_ref and y_ref must have the same dimensions. + The inputted slope is used even in the case of pd.Series or np.ndarray inputs as the purpose of this function + is to perform a linear transformation using a predefined slope, not to fit a line to the input data. + + :param x_target: Target x value(s) at which to calculate y_target. + :type x_target: float or int or numpy.ndarray or pandas.Series + :param x_ref: Reference x value(s) of known reference point(s) on the line. + :type x_ref: float or int or numpy.ndarray or pandas.Series + :param y_ref: Reference y value(s) of known reference point(s) on the line. + :type y_ref: float or int or numpy.ndarray or pandas.Series + :param slope: Slope of the line equal to (y_target - y_ref) / (x_target - x_ref) where + ref and target x and y are any two points on the line. + :type slope: float or int + :return: Value(s) of y_target at specified x_target. + :rtype: float or int or numpy.ndarray or pandas.Series + + **Example usage** + :: + import brightwind as bw + + data = bw.load_csv(bw.demo_datasets.demo_data) + + # calculate y_target for x_target = 11 where (x_ref, y_ref) = (5, 10) is a point on the line and the slope is -0.5. + bw.utils.utils.linear_transform(x_target=11, x_ref=5, y_ref=10, slope=-0.5) + # 7.0 + + # calculate y_target for x_target as a pandas.Series and (x_ref, y_ref) = (2, 20) is a point on the line + # and the slope is -0.0065. + bw.utils.utils.linear_transform( + x_target=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065) + + # calculate y_target for x_target and x_ref as a int and y_ref as a np.ndarray where (x_ref, y_ref) = (2, y_ref) + # are points on the line and the slope is -0.0065. + bw.utils.utils.linear_transform( + x_target=20, + x_ref=2, + y_ref=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) + """ + # check input types + for var, var_name in zip([x_ref, y_ref, x_target], ['x_ref', 'y_ref', 'x_target']): + if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series) or + isinstance(var, np.ndarray)): + raise TypeError(f"{var_name} must be a float or int or a numpy.ndarray or pandas.Series.") + + if not (isinstance(slope, float) or isinstance(slope, int)): + raise TypeError("slope must be a float or int.") + + # check dimensions of x_ref and x_target if arrays or Series + if (isinstance(x_ref, np.ndarray) or isinstance(x_ref, pd.Series)) and ( + isinstance(x_target, np.ndarray) or isinstance(x_target, pd.Series)): + if len(x_ref) != len(x_target): + raise ValueError("x_ref and x_target must have the same dimensions.") + + # check dimensions of y_ref if arrays or Series + if isinstance(x_target - x_ref, np.ndarray) or isinstance(x_target - x_ref, pd.Series): + if isinstance(y_ref, np.ndarray) or isinstance(y_ref, pd.Series): + if len(y_ref) != len(x_target - x_ref): + raise ValueError("y_ref must have the same dimensions as x_target or x_ref.") + + y_target = slope * (x_target - x_ref) + y_ref + + return y_target diff --git a/tests/test_analyse.py b/tests/test_analyse.py index c1857de5..199ecc58 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -697,3 +697,48 @@ def test_scale_air_pressure_to_height(): ref_air_temp_degC=pd.Series([1, 2]), ref_height_m=2, target_height_m=10) + +def test_scale_air_density_to_height(): + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) == 1.22174 + + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, + ref_height_m=80, + target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) == 1.22 + + test_density = bw.calc_air_density(DATA.T2m, DATA.P2m) + pd.testing.assert_series_equal(bw.scale_air_density_to_height( + ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data = [1.186780, 1.187175, 1.187747, 1.185950, 1.186301 , 1.185686], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + +def test_scale_air_temperature_to_height(): + assert bw.scale_air_temperature_to_height(10.0065, 10, 11) == 10.0 + + assert bw.scale_air_temperature_to_height(ref_air_temperature=10, + ref_height_m=12, + target_height_m=10, + lapse_rate_deg_m=-0.001) == 10.002 + + pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20, + lapse_rate_deg_m=-0.001), + pd.Series(data=[0.936, 0.845, 0.713, 0.834, 0.753, 0.895], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20), + pd.Series(data=[0.837, 0.746, 0.614, 0.735, 0.654, 0.796], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) diff --git a/tests/test_utils.py b/tests/test_utils.py index 36c85a46..454a2b6c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ import pytest import brightwind as bw import pandas as pd +import numpy as np DATA = bw.load_csv(bw.demo_datasets.demo_data) DATA = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) @@ -25,3 +26,58 @@ def test_slice_data(): data_sliced = bw.utils.utils.slice_data(DATA, date_to='2017-10-23') assert data_sliced.index[0] == DATA.index[0] + +def test_linear_transform(): + # test with float and int inputs + assert bw.utils.utils.linear_transform(x_target=10, x_ref=5, y_ref=10, slope=-0.5) == 7.5 + + # test with array input for y_ref + assert (bw.utils.utils.linear_transform( + x_target=20, + x_ref=2, + y_ref=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) == np.array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796])).all() + + # test with pandas Series input for y_ref + pd.testing.assert_series_equal(bw.utils.utils.linear_transform( + x_target=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065).round(3), + pd.Series(data = [20.007, 20.007, 20.008, 20.007, 20.008, 20.007], + index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + # test error raising for invalid input types + with pytest.raises(TypeError): + bw.utils.utils.linear_transform( + x_target='invalid_type', + x_ref=2, + y_ref=20, + slope=-0.0065) + + #test error raising for invalid slope type + with pytest.raises(TypeError): + bw.utils.utils.linear_transform( + x_target=10, + x_ref=2, + y_ref=20, + slope='invalid_type') + + #test error raising for mismatched dimensions + with pytest.raises(ValueError): + bw.utils.utils.linear_transform( + x_target=np.array([1, 2, 3]), + x_ref=np.array([1, 2]), + y_ref=20, + slope=-0.5) + + #test error raising for mismatched dimensions with Series + with pytest.raises(ValueError): + bw.utils.utils.linear_transform( + x_target=pd.Series([1, 2, 3]), + x_ref=2, + y_ref=pd.Series([1, 2]), + slope=-0.5) + From 1ec87cf36b2176e8835371588da0ebfd32832800 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:43:45 +0100 Subject: [PATCH 14/27] [Iss535] updated calc_air_density to include humidity and follow IEC (#538) * iss #530 added lapse_temp function * iss #530 updated tests and docstring * iss #530 updated docstring * iss #530 minor docstring update * iss #531 added tests and updated function name to lapse_temperature * Revert "iss #531 added tests and updated function name to lapse_temperature" This reverts commit d41ec28e0b4f7b9e1ecf6f2ff85b8312cbe196e4. * iss #530 updated function name to lapse_temperature * iss #530 added tests * iss #534 added lapse_density function * iss #534 added tests * iss #530 updated changelog * iss #534 updated changelog * iss #530 added private function for lapsing, bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 updated name to extrapolate_temperature and updated to use bw.utils.utils.vertically_extrapolate_at_constant_rate * iss #530 changed lapse_density to extrapolate_density * iss #535 updated calc_air_density to include humidity and follow IEC-recommended method * iss #530 addressed PR comments, updated function names * iss #530 small formatting fixes * iss #530 updated reference to ref for consistency with other PR * iss #530 formatting and docstring and naming fixes * iss #530 updated functions to include type for each argument * iss #535 attempt to avoid breaking change * iss 530 changed linear funct to public and fixed several typos and example issues * iss 530 fixed tests for utils and analyse * iss 530 fixed typo changelog * iss 530 updated linear_transform to raise errors and added tests * iss 530 added more tests * iss #530 added source of air density lapse rate to docstring * iss #530 include reference for air density lapse rate in docstring * iss #535 addressed comments and added workaround to avoid breaking change * iss #535 updated tests * iss #535 removed all breaking changes * iss #535 inor format change * iss #535 fix bug for dataframe input * iss #535 refactor tests * iss #535 updated tests and examples to match * iss #535 fixed bug for rel_humidity_percent * iss #535 updated constants in analyse.py * iss #535 docstring and formatting fixes * iss #535 fixed line length --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi --- CHANGELOG.md | 5 +- brightwind/analyse/analyse.py | 242 ++++++++++++++++++++++++++++------ tests/test_analyse.py | 55 ++++++-- 3 files changed, 248 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54ec8e15..66f023b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,9 @@ Additional labels for pre-release and build metadata are available as extensions ### New Features and Enhancements 1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) -1. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([[#534](https://github.com/brightwind-dev/brightwind/issues/534)]) -1. Added `scale_air_temperature_to_height` to output a air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([[#530](https://github.com/brightwind-dev/brightwind/issues/530)]) +2. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) +3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) +4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 3dbd5fb0..061b6ed2 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -42,6 +42,8 @@ # Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere GAS_CONST_DRY_AIR = 287.05 +# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) +AIR_DENSITY_LAPSE_RATE = -0.113 def dist_matrix(var_series, x_series, y_series, num_bins_x=None, num_bins_y=None, @@ -2026,64 +2028,217 @@ def sector_ratio(wspd_1, wspd_2, wdir, sectors=72, min_wspd=3, direction_bin_arr return fig -def calc_air_density(temperature, pressure, elevation_ref=None, elevation_site=None, lapse_rate=-0.113, - specific_gas_constant=286.9): +def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], + pressure: Union[float, pd.Series, pd.DataFrame], + elevation_ref: Union[float, int] = None, + elevation_site: Union[float, int] = None, + lapse_rate: float = AIR_DENSITY_LAPSE_RATE, + specific_gas_constant: float = 286.9, + rel_humidity_percent: Union[float, pd.Series, pd.DataFrame] = None + ) -> Union[float, pd.Series]: """ - Calculates air density for a given temperature and pressure and extrapolates that to the site if both reference - and site elevations are given. - - :param temperature: Temperature values in degree Celsius - :type temperature: float or pandas.Series or pandas.DataFrame - :param pressure: Pressure values in hectopascal, hPa, (1,013.25 hPa = 101,325 Pa = 101.325 kPa = - 1 atm = 1013.25 mbar) - :type pressure: float or pandas.Series or pandas.DataFrame - :param elevation_ref: Elevation, in meters, of the reference temperature and pressure location. - :type elevation_ref: Floating point value (decimal number) - :param elevation_site: Elevation, in meters, of the site location to calculate air density for. - :type elevation_site: Floating point values (decimal number) - :param lapse_rate: Air density lapse rate kg/m^3/km, default is -0.113 - :type lapse_rate: Floating point value (decimal number) - :param specific_gas_constant: Specific gas constant, R, for humid air J/(kg.K), default is 286.9 - :type specific_gas_constant: Floating point value (decimal number) - :return: Air density in kg/m^3 - :rtype: float or pandas.Series depending on the input + Calculates air density for a given air temperature (temperature), air pressure (pressure) + and relative humidity (rel_humidity_percent) using method outlined in IEC Standard (61400-12-1). + + Note that the value of the specific_gas_constant argument is used only when rel_humidity_percent is None. + This specific_gas_constant argument and its default value of 286.9 are retained to maintain backwards + compatibility for when rel_humidity_percent is None. + However, if rel_humidity_percent is not None, the specific_gas_constant is ignored as the air density + calculation which incorporates humidity depends on the two specific gas constants for dry air + (287.05 J/kg*K) and water vapour (461.5 J/kg*K) and therefore the specific_gas_constant argument is redundant. + + WARNING: The `specific_gas_constant` argument will be removed in a future 3.0 release of brightwind. + + Function gives option to scale air density to the elevation of the site if both the reference elevation + (elevation_ref) and site elevation (elevation_site) are provided. This scaling assumes that air density + decreases with increasing altitude above the earth's surface - a default air density lapse rate of -0.113 kg/m3/km + is adopted. Taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) + + WARNING: Option of scaling air density to height will be removed in a future 3.0 release of brightwind. Please call + `scale_air_density_to_height()` separately instead. + + :param temperature: Air temperature values in degree Celsius + :type temperature: float or pandas.Series or pandas.DataFrame + :param pressure: Air pressure values in hectopascal, hPa or millibar, mbar (Note 1hPa = 1mbar) + (1013.25 hPa = 101,325 Pa = 101.325 kPa = 1 atm = 1013.25 mbar) + :type pressure: float or pandas.Series or pandas.DataFrame + :param elevation_ref: Elevation, in meters, of the reference air density location (and air temperature, + air pressure, relative humidity measurements used for the calculation). + :type elevation_ref: float or int + :param elevation_site: Elevation, in meters, of the site location for which air density is calculated. + Scaled air density value(s) are calculated from elevation_ref to elevation_site + if both are specified. + :type elevation_site: float or int + :param lapse_rate: Air density lapse rate kg/m^3/km. Default is -0.113 kg/m^3/km. + :type lapse_rate: float + :param specific_gas_constant: Specific gas constant R, for moist air, in J/(kg*K). Default is 286.9 J/(kg*K). + If rel_humidity_percent is not None, this argument is ignored and the specific + gas constant for dry air of 287.05 J/(kg*K) is used instead. + If rel_humidity_percent is None, then provided the user is unconcerned with + backwards compatibility, the user should set the specific_gas_constant to 287.05 or + set rel_humidity_percent to 0 to use the specific gas constants for dry air. + :type specific_gas_constant: float + :param rel_humidity_percent: Relative humidity values as a percentage. Default is None. If None, the air density + calculation ignores humidity. + :type rel_humidity_percent: float or pandas.Series or pandas.DataFrame + :return: Air density in kg/m^3. Output type depends on type(temperature), type(pressure), + type(rel_humidity_percent) provided. If all inputs are float, output is float. + If any input is pandas.Series or pandas.DataFrame, output is pandas.Series. + :rtype: float or pandas.Series **Example usage** :: - import brightwind as bw + import brightwind as bw - #For a series of air densities - data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_site_data) - air_density = bw.calc_air_density(data.T2m, data.P2m) + data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) + + # Calculate air density from input air temperature and air pressure series + # (rel_humidity_percent=None and specific_gas_constant=286.9 by default) + bw.calc_air_density(data.T2m, data.P2m).head(5) + # Timestamp + # 2016-01-09 15:30:00 1.190011 + # 2016-01-09 15:40:00 1.190363 + # 2016-01-09 17:00:00 1.186939 + # 2016-01-09 17:10:00 1.187684 + # 2016-01-09 17:20:00 1.188079 + # dtype: float64 - #For a single value - bw.calc_air_density(15, 1013) + # Calculate air density from input air temperature and air pressure series + # (rel_humidity_percent=None and specific_gas_constant=287.05 which is the constant for dry air) + bw.calc_air_density(data.T2m, data.P2m, specific_gas_constant=287.05).head(5) + # Timestamp + # 2016-01-09 15:30:00 1.189389 + # 2016-01-09 15:40:00 1.189741 + # 2016-01-09 17:00:00 1.186319 + # 2016-01-09 17:10:00 1.187064 + # 2016-01-09 17:20:00 1.187458 + # dtype: float64 - #For a single value with ref and site elevation - bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) + # Calculate air density from single float values of air temperature and air pressure + # (rel_humidity_percent=None and specific_gas_constant=286.9 by default) + bw.calc_air_density(15, 1013).round(5) + # 1.22535 - """ + # Calculate air density from single float value of air temperature and air pressure + # (rel_humidity_percent=None and specific_gas_constant=287.05 which is the constant for dry air) + bw.calc_air_density(15, 1013, specific_gas_constant=287.05).round(5) + # 1.22471 - temp = temperature - temp_kelvin = temp + 273.15 # to convert deg C to Kelvin. - pressure = pressure * 100 # to convert hPa to Pa - ref_air_density = pressure / (specific_gas_constant * temp_kelvin) + # Calculate air density from input air temperature, air pressure and relative humidity series + # As rel_humidity_percent is not None, gas constant for dry air of 287.05 is used + bw.calc_air_density(data.T2m, data.P2m, rel_humidity_percent=data.RH2m).head(5) + # Timestamp + # 2016-01-09 15:30:00 1.186163 + # 2016-01-09 15:40:00 1.186530 + # 2016-01-09 17:00:00 1.183012 + # 2016-01-09 17:10:00 1.183790 + # 2016-01-09 17:20:00 1.184202 + # dtype: float64 + + # Calculate air density from single float value of air temperature, air pressure, relative humidity + # As rel_humidity_percent is not None, gas constant for dry air of 287.05 is used + bw.calc_air_density(15, 1013, rel_humidity_percent=50).round(5) + # 1.22093 + + # Calculate air density from a single float value of air temperature, air pressure, relative humidity + # and scale result from reference elevation to site elevation. + bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200) + # 1.198 + + """ + # Convert DataFrame inputs to Series + if isinstance(temperature, pd.DataFrame): + temperature = _convert_df_to_series(temperature).copy() + if isinstance(pressure, pd.DataFrame): + pressure = _convert_df_to_series(pressure).copy() + if isinstance(rel_humidity_percent, pd.DataFrame): + rel_humidity_percent = _convert_df_to_series(rel_humidity_percent).copy() + + # Check dimensions of temperature, pressure and rel_humidity_percent if not float or int + if isinstance(temperature, pd.Series) and (isinstance(pressure, pd.Series)): + if len(temperature) != len(pressure): + raise ValueError("temperature and pressure must have the same dimensions.") + if isinstance(rel_humidity_percent, pd.Series): + if len(temperature) != len(rel_humidity_percent): + raise ValueError("temperature, pressure and rel_humidity_percent must have the same dimensions.") + + lapse_rate_per_m = lapse_rate * 0.001 # convert lapse rate from kg/m3/km to kg/m3/m + temp_K = temperature + 273.15 # to convert deg C to Kelvin. + press_Pa = pressure * 100 # to convert hPa to Pa + vapour_press = 0.0000205 * np.exp(0.0631846 * temp_K) + specific_gas_constant_water = 461.5 + specific_gas_constant_dry_air = GAS_CONST_DRY_AIR + + if rel_humidity_percent is None: + # If relative humidity is None, the old method ignoring relative humidity is used. + # To avoid a breaking change, the value of the specific_gas_constant argument is used. + # (Default specific_gas_constant is 286.9 which is for moist air) + # If a user is not concerned with backwards compatibility, the value of this argument should be updated to match + # the specific gas constant for dry air (287.05) which makes more physical sense given relative humidity is + # taken as zero in the calculation below. + gas_constant_air = specific_gas_constant + rel_hum_decimal = 0.0 + # Deprecation warning + warnings.warn( + ( + "\nThe `specific_gas_constant` argument of `calc_air_density()` will be removed in a future 3.0 " + "release of brightwind.\nNote that this value is used only when `rel_humidity_percent` is None." + "\nPlease set `rel_humidity_percent` to 0 instead if you want to use the specific gas constant " + "for dry air (287.05 J/kg*K)." + ), + DeprecationWarning, + stacklevel=2 + ) - if elevation_ref is not None and elevation_site is not None: - site_air_density = round(ref_air_density + (((elevation_site - elevation_ref) / 1000) * lapse_rate), 3) - return site_air_density - elif elevation_site is None and elevation_ref is not None: - raise TypeError('elevation_site should be a number') - elif elevation_site is not None and elevation_ref is None: - raise TypeError('elevation_ref should be a number') else: - return ref_air_density + # Otherwise, if the rel_humidity_percent is not None (including = 0), the new method including relative humidity + # is used. The specific_gas_constant_dry_air of 287.05 is used as the air density calculation incorporating + # relative humidity is clearly split into a dry component and a moist component. Therefore it makes physical + # sense to use the specific gas constant for dry air in this context. + gas_constant_air = specific_gas_constant_dry_air + rel_hum_decimal = rel_humidity_percent * 0.01 + + # Contribution from dry air + dry_air_term = press_Pa / gas_constant_air + + # Contribution from water vapour + water_vapour_term = rel_hum_decimal * vapour_press * ( + (1 / gas_constant_air) - (1 / specific_gas_constant_water)) + + # Combine to get air density + air_density = (1 / temp_K) * (dry_air_term - water_vapour_term) + + if elevation_ref is not None and elevation_site is not None: + # Deprecation warning + warnings.warn( + ( + "\nScaling air density to height within `calc_air_density()` is deprecated and will be removed in a " + "future 3.0 release of brightwind.\nPlease call `scale_air_density_to_height()` separately instead. \n" + ), + DeprecationWarning, + stacklevel=2 + ) + # Perform extrapolation + scaled_air_density = scale_air_density_to_height(air_density, + elevation_ref, + elevation_site, + lapse_rate_per_m) + return round(scaled_air_density, 3) + + # Validate elevation arguments + if elevation_site is not None and elevation_ref is None: + raise TypeError("Specify value of elevation_ref (float or int) when elevation_site is provided.") + elif elevation_ref is not None and elevation_site is None: + raise TypeError("Specify value of elevation_site (float or int) when elevation_ref is provided.") + + return air_density def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], ref_air_temp_degC: Union[float, pd.Series], ref_height_m: Union[float, int], - target_height_m: Union[float, int]) -> Union[float, int, pd.Series]: + target_height_m: Union[float, int] + ) -> Union[float, int, pd.Series]: """ Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). @@ -2166,7 +2321,8 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], ref_height_m: float, target_height_m: float, - lapse_rate_kg_m3_m: float = -0.000113) -> Union[float, pd.Series]: + lapse_rate_kg_m3_m: float = (0.001 * AIR_DENSITY_LAPSE_RATE) + ) -> Union[float, pd.Series]: """ Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 199ecc58..a376c297 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -632,20 +632,57 @@ def test_ti_by_sector(): def test_calc_air_density(): - bw.calc_air_density(DATA[['T2m']], DATA[['P2m']]) - bw.calc_air_density(DATA.T2m, DATA.P2m) - bw.calc_air_density(DATA.T2m, DATA.P2m, elevation_ref=0, elevation_site=200) + # test DataFrame input + assert (bw.calc_air_density(DATA[['T2m']], DATA[['P2m']]).dropna().values == + bw.calc_air_density(DATA.T2m, DATA.P2m).dropna().values).all() + assert (round(bw.calc_air_density(pd.Series([15, 12.5, -5, 23]), pd.Series([1013, 990, 1020, 900])), 6 + ) == pd.Series([1.225350, 1.208010, 1.325842, 1.059254])).all() + + # test Series inputs + assert list(round(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5), 6).values + ) == [1.199177, 1.199838, 1.199794, 1.199439, 1.201066] + assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5).values - + pd.Series([1.19918, 1.19984, 1.19979, 1.19944, 1.20107])) < 1e-3).all() + assert (abs(bw.calc_air_density(DATA.T2m, DATA.P2m, rel_humidity_percent=DATA.RH2m).tail(5).values - + pd.Series([1.19529, 1.19601, 1.19592, 1.19555, 1.19719])) < 1e-3).all() + assert list(bw.calc_air_density(DATA.T2m, DATA.P2m, specific_gas_constant=287.05).head(5).round(6).dropna() + ) == [1.187064, 1.187458] + + #t test Series inputs with elevation adjustment + assert list(bw.calc_air_density(DATA.T2m, DATA.P2m, elevation_ref=0, elevation_site=200).tail(5).values + ) == [1.177, 1.177, 1.177, 1.177, 1.178] + + # test float/int inputs + assert bw.calc_air_density(15, 1013) == 1.2253503331640465 + assert (bw.calc_air_density(15, 1012, rel_humidity_percent=None, specific_gas_constant=287.05) == + bw.calc_air_density(15, 1012, rel_humidity_percent=0)) + assert round(bw.calc_air_density(15, 1012, rel_humidity_percent=0), 5) == 1.2235 + assert round(bw.calc_air_density(15, 1012), 5) == 1.22414 + assert abs(bw.calc_air_density(15, 1013, specific_gas_constant=287.05) - 1.22471) < 1e-3 + assert abs(bw.calc_air_density(15, 1013, rel_humidity_percent=50) - 1.22093) < 1e-3 + assert round(bw.calc_air_density(15, 1013, rel_humidity_percent=50, + elevation_ref=0, elevation_site=200), 5) == 1.198 + + # test float/int inputs with elevation adjustment + assert bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) == 1.203 + assert bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200 + ) - bw.scale_air_density_to_height( + bw.calc_air_density(15, 1013, rel_humidity_percent=50), 0, 200) <1e-3 + + # test errors with pytest.raises(TypeError) as except_info: bw.calc_air_density(15, 1013, elevation_site=200) - assert str(except_info.value) == 'elevation_ref should be a number' + assert str(except_info.value) == "Specify value of elevation_ref (float or int) when elevation_site is provided." with pytest.raises(TypeError) as except_info: bw.calc_air_density(15, 1013, elevation_ref=200) - assert str(except_info.value) == 'elevation_site should be a number' - assert abs(bw.calc_air_density(15, 1013) - 1.225) < 1e-3 - assert abs(bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) - 1.203) < 1e-3 - assert (abs(bw.calc_air_density(pd.Series([15, 12.5, -5, 23]), pd.Series([1013, 990, 1020, 900])) - - pd.Series([1.225, 1.208, 1.326, 1.059])) < 1e-3).all() + assert str(except_info.value) == "Specify value of elevation_site (float or int) when elevation_ref is provided." + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) + assert str(except_info.value) == "temperature and pressure must have the same dimensions." + with pytest.raises(ValueError) as except_info: + bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) + assert str(except_info.value) == "temperature, pressure and rel_humidity_percent must have the same dimensions." def test_dist_matrix_by_direction_sector(): From 9ef75ff74fab39eba3e487012185a7f2bbf106c8 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:23:43 +0100 Subject: [PATCH 15/27] [Iss541] update scale wind spd name (#542) * iss #541 renamed scale_wind_speed as apply_scale_factor, added tests and updated changelog * iss #541 updated all references to scale_wind_speed to new name * Revert "iss #541 updated all references to scale_wind_speed to new name" This reverts commit 55aa006a2e10b2c76f41eb7f8782eb2577af626a. * Revert "iss #541 renamed scale_wind_speed as apply_scale_factor, added tests and updated changelog" This reverts commit 7d2905a39d0193fb99b8e488109c23a212d2a70f. * iss #541 addressed Dan's comments, updated existing scale_wind_speed docstring to remove inaccuracy * iss #541 added apply_scale_factor to utils * iss #541 updated to allow pd.DataFrame * iss #541 updated tests and docstrings * iss #541 minor update changelog * iss #541 updates for type, wording * iss #541 minor fix * iss #541 updated changelog * iss #541 added error if series not numeric * iss #541 updated to raise value error if input is non-numeric np.ndarray * iss #541 fix PEP8 in utils * iss #541 fix PEP 8 plus some text * iss #541 add return type and fix test --------- Co-authored-by: Sara Rafter Co-authored-by: BiancaMorandi Co-authored-by: stephenholleran --- CHANGELOG.md | 1 + brightwind/analyse/analyse.py | 5 +-- brightwind/transform/transform.py | 59 +++++++++++++++++++++----- brightwind/utils/utils.py | 69 ++++++++++++++++++++++++++++++- tests/test_transform.py | 10 +++++ tests/test_utils.py | 10 ++++- 6 files changed, 139 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f023b9..55720797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Additional labels for pre-release and build metadata are available as extensions 2. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) 3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) 4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) +5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 061b6ed2..3844374b 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -6,7 +6,6 @@ from brightwind.utils.utils import _convert_df_to_series from brightwind.utils.utils import validate_coverage_threshold from brightwind.export.export import _calc_mean_speed_of_freq_tab -from brightwind.transform.transform import scale_wind_speed import matplotlib.pyplot as plt import warnings import textwrap @@ -1304,7 +1303,7 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 # This scale factor is used to correct `var_series` when concurrent with `direction_series`. if target_freq_table_mean is not None: scale_factor = target_freq_table_mean / data_concurrent[var_series.name].mean() - var_series_scaled = scale_wind_speed(data_concurrent[var_series.name], scale_factor) + var_series_scaled = tf.scale_wind_speed(data_concurrent[var_series.name], scale_factor) else: var_series_scaled = var_series @@ -1339,7 +1338,7 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 else: freq_tab_mean = _calc_mean_speed_of_freq_tab(result) scale_factor = target_freq_table_mean / freq_tab_mean - var_series_scaled = scale_wind_speed(var_series_scaled, scale_factor) + var_series_scaled = tf.scale_wind_speed(var_series_scaled, scale_factor) abs_percentage_diff = abs(100 * (target_freq_table_mean - freq_tab_mean) / freq_tab_mean) if abs_percentage_diff > 0.01: diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 1d301593..edcf116c 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1,12 +1,15 @@ -import datetime -import numpy as np +import copy +import datetime +import warnings +from typing import Union + +import numpy as np import pandas as pd + from brightwind.utils import utils from brightwind.load.station import _Measurements from brightwind.load.station import DATE_INSTEAD_OF_NONE from brightwind.utils.utils import validate_coverage_threshold -import copy -import warnings __all__ = ['average_data_by_period', 'merge_datasets_by_period', @@ -994,17 +997,53 @@ def apply_wspd_slope_offset_adj(data, measurements, inplace=False): return df -def scale_wind_speed(spd, scale_factor: float): +def scale_wind_speed(spd: Union[pd.Series, pd.DataFrame, float, int], + scale_factor: Union[int, float] + ) -> Union[pd.Series, pd.DataFrame, float, int]: """ Scales wind speed by the scale_factor - :param spd: Series or data frame or a single value of wind speed to scale - :param scale_factor: Scaling factor in decimal, if scaling factor is 0.8 output would be (1+0.8) times wind speed, - if it is -0.8 the output would be (1-0.8) times the wind speed - :return: Series or data frame with scaled wind speeds + :param spd: Wind speed value(s) to scale. + :type spd: pandas.Series or pandas.DataFrame or float or int + :param scale_factor: Scaling factor to use for scaling wind speed. + If scaling factor is 0.8, output would be 0.8 times wind speed. + :type scale_factor: int or float + :return: Value(s) of scaled wind speed. Output type depends on type(spd). + :rtype: pandas.Series or pandas.DataFrame or float or int + **Example usage** + :: + import brightwind as bw + import pandas as pd + import numpy as np + + data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_data) + + # scale float by scale_factor of 0.5 + bw.scale_wind_speed(3, 0.5) + # 1.5 + + # scale np.array by scale_factor of 0.5 + bw.scale_wind_speed(np.array([0, 1, 2]), 0.5) + # array([0. , 0.5, 1. ]) + + # scale pd.Series by scale_factor of 0.5 + bw.scale_wind_speed(data['Spd40mN'].tail(3), 0.5) + # Timestamp + # 2017-11-23 10:30:00 4.0150 + # 2017-11-23 10:40:00 3.4055 + # 2017-11-23 10:50:00 2.9325 + # Name: Spd40mN, dtype: float64 + + # scale pd.DataFrame by scale factor of 2 + bw.scale_wind_speed(data.tail(3)[['Spd60mS', 'Spd40mS']], 2) + # Spd60mS Spd40mS + # Timestamp + # 2017-11-23 10:30:00 16.900 15.750 + # 2017-11-23 10:40:00 14.318 13.336 + # 2017-11-23 10:50:00 12.808 11.498 """ - return spd * scale_factor + return utils.apply_scale_factor(spd, scale_factor) def offset_wind_direction(wdir, offset: float): diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index bd6d6361..83adf1c6 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -10,7 +10,8 @@ 'is_file', 'is_file_extension', 'validate_json', - 'linear_transform'] + 'linear_transform', + 'apply_scale_factor'] def _range_0_to_360(direction): @@ -339,3 +340,69 @@ def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], y_target = slope * (x_target - x_ref) + y_ref return y_target + + +def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], + scale_factor: Union[float, int] + ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: + """ + Scales data by the scale_factor. + + If data input is pd.DataFrame, only numeric columns are scaled. + + :param data: Data value(s) to scale by the scale_factor. + :type data: float or int or pandas.Series or pandas.DataFrame or numpy.array + :param scale_factor: Scaling factor to use for scaling data values. + :type scale_factor: float or int + :returns: Scaled data value(s). Output type depends on type(data). + :rtype: float or int or pandas.Series or pandas.DataFrame or numpy.array + + **Example usage** + :: + import brightwind as bw + import pandas as pd + import numpy as np + + # scale float by scale_factor of 0.5 + bw.utils.utils.apply_scale_factor(3, 0.5) + # 1.5 + + # # scale np.array by scale_factor of 0.5 + # bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) + # array([0. , 0.5, 1. ]) + + # scale pd.Series by scale_factor of -10 + bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) + # 0 -100 + # 1 -200 + # 2 -300 + # 3 -400 + # dtype: int64 + + # scale pd.DataFrame by scale factor of 2 + df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) + bw.utils.utils.apply_scale_factor(df, 2) + # a b c + # 0 1.0 6 a + # 1 2.4 8 b + """ + + if not isinstance(data, (float, int, pd.DataFrame, pd.Series, np.ndarray)): + raise ValueError('data should be a float or int or pd.DataFrame or pd.Series or np.ndarray') + if not isinstance(scale_factor, (float, int)): + raise ValueError('scale_factor should be a float or int') + + if isinstance(data, pd.DataFrame): + # only apply scaling to numeric columns + numeric_df = scale_factor * (data.select_dtypes(include='number')) + result = pd.concat([numeric_df, data.select_dtypes(exclude='number')], axis=1) + return result + else: + if isinstance(data, pd.Series): + if not pd.api.types.is_numeric_dtype(data): + raise ValueError('data inputted as a pandas.Series must be numeric') + if isinstance(data, np.ndarray): + if not np.issubdtype(data.dtype, np.number): + raise ValueError('data inputted as a np.ndarray must be numeric') + + return scale_factor * data diff --git a/tests/test_transform.py b/tests/test_transform.py index 7e3f176d..90a37fec 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -799,3 +799,13 @@ def test_merge_datasets_by_period(): assert round(mrgd_data['Spd80mN_Coverage'].values[0], 8) == 0.00179211 + +def test_scale_wind_speed(): + + assert bw.scale_wind_speed(3, 0.5) == 1.5 + assert (bw.scale_wind_speed(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() + assert (bw.scale_wind_speed(DATA_CLND['Spd40mN'].tail(3), 0.5)==[4.015 , 3.4055, 2.9325]).all() + + df = pd.DataFrame({'Spd_100m':[0.5, 1.2], 'Spd_100m':[3, 4], 'c':['a', 'b']}) + result_df = pd.DataFrame({'Spd_100m':[1.0, 2.4], 'Spd_100m':[6, 8], 'c':['a', 'b']}) + assert result_df.equals(bw.scale_wind_speed(df, 2)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 454a2b6c..a188573f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,9 @@ import pytest -import brightwind as bw import pandas as pd import numpy as np +import brightwind as bw + DATA = bw.load_csv(bw.demo_datasets.demo_data) DATA = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) WSPD_COLS = ['Spd80mN', 'Spd80mS', 'Spd60mN', 'Spd60mS', 'Spd40mN', 'Spd40mS'] @@ -81,3 +82,10 @@ def test_linear_transform(): y_ref=pd.Series([1, 2]), slope=-0.5) +def test_apply_scale_factor(): + assert bw.utils.utils.apply_scale_factor(3, 0.5) == 1.5 + assert (bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() + assert (bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) == [-100, -200, -300, -400]).all() + df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) + result_df = pd.DataFrame({'a':[1.0, 2.4], 'b':[6, 8], 'c':['a', 'b']}) + assert result_df.equals(bw.utils.utils.apply_scale_factor(df, 2)) From b270f3d7aed43cb1155ca48fd6d65c2b37353ea6 Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:48:45 +0000 Subject: [PATCH 16/27] Moved scaling functions to transform/scale.py (SR1) (#548) * iss #544 moved scaling functions to transform/scale.py * iss #544 updated paths in examples * iss #544 reordered functions * iss #544 PEP8 formatting * iss #544 fix indent error + PEP8 * iss #544 fixed flake8 issues in scale.py and test_transform.py * iss #544 fixed flake8 in test_scale.py --------- Co-authored-by: stephenholleran --- brightwind/__init__.py | 1 + brightwind/analyse/analyse.py | 228 +---------------- brightwind/transform/scale.py | 390 ++++++++++++++++++++++++++++++ brightwind/transform/transform.py | 3 +- brightwind/utils/utils.py | 152 +----------- tests/test_analyse.py | 86 +------ tests/test_scale.py | 160 ++++++++++++ tests/test_transform.py | 8 +- tests/test_utils.py | 63 ----- 9 files changed, 565 insertions(+), 526 deletions(-) create mode 100644 brightwind/transform/scale.py create mode 100644 tests/test_scale.py diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 20152832..5814a0f6 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -5,6 +5,7 @@ from .analyse.analyse import * from .analyse.plot import * from .transform.transform import * +from .transform.scale import * from .export.export import * from . import demo_datasets from .utils.gis import * diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 3844374b..0c5d746c 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -3,6 +3,7 @@ from brightwind.transform import transform as tf from brightwind.utils import utils from brightwind.analyse import plot as bw_plt +from brightwind.transform.scale import scale_air_density_to_height from brightwind.utils.utils import _convert_df_to_series from brightwind.utils.utils import validate_coverage_threshold from brightwind.export.export import _calc_mean_speed_of_freq_tab @@ -27,16 +28,7 @@ 'basic_stats', 'TI', 'sector_ratio', - 'calc_air_density', - 'scale_air_pressure_to_height', - 'scale_air_density_to_height', - 'scale_air_temperature_to_height'] - -# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere -ACCEL_DUE_TO_GRAVITY = 9.80665 - -# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere -TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + 'calc_air_density'] # Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere GAS_CONST_DRY_AIR = 287.05 @@ -2219,9 +2211,9 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], ) # Perform extrapolation scaled_air_density = scale_air_density_to_height(air_density, - elevation_ref, - elevation_site, - lapse_rate_per_m) + elevation_ref, + elevation_site, + lapse_rate_per_m) return round(scaled_air_density, 3) # Validate elevation arguments @@ -2233,213 +2225,3 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], return air_density -def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], - ref_air_temp_degC: Union[float, pd.Series], - ref_height_m: Union[float, int], - target_height_m: Union[float, int] - ) -> Union[float, int, pd.Series]: - """ - Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) - and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). - - Calculation based on ISO:2533-1975 Standard Atmosphere (https://www.iso.org/obp/ui/#iso:std:iso:2533:en) - as suggested by IEC 61400-12-1: - - scaled_air_pressure = ref_air_pressure_hPa*((1 + (L/ref_air_temp_K)*(target_height_m - ref_height_m))**(-g/(L*R))) - where: - g = 9.80665 is acceleration due to gravity (m/s^2) - L = -0.0065 is the temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) - R = 287.05 is the specific gas constant for dry air (J/K/kg or m2/K/s2) - - :param ref_air_pressure_hPa: Reference air pressure value(s) in hPa (1mbar = 1hPa = 100Pa). - :type ref_air_pressure_hPa: float or int or pandas.Series - :param ref_air_temp_degC: Reference air temperature value(s) in degrees celsius. - :type ref_air_temp_degC: float or int or pandas.Series - :param ref_height_m: Height (in metres) of reference air temperature (ref_air_temp_degC) - and air pressure (ref_air_pressure_hPa). - :type ref_height_m: float or int - :param target_height_m: Height (in metres) which ref_air_pressure_hPa is scaled to. - :type target_height_m float or int - :return: Air pressure at specified height of target_height_m in hPa (1mbar = 1hPa = 100Pa). - Type depends on type(ref_air_pressure_hPa) and type(ref_air_temp_degC) inputs. - :rtype: float or int or pandas.Series - - **Example usage** - :: - - import brightwind as bw - - # scale float value of air pressure - round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, ref_height_m=10, - target_height_m=200), 2) - # 977.45 - - # scale air pressure based on input series of reference air pressure and air temperature - data = bw.load_csv(bw.demo_datasets.demo_data) - - bw.scale_air_pressure_to_height(ref_air_pressure_hPa=data['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_air_temp_degC=data['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10).round(2) - # Timestamp - # 2016-01-09 17:10:00 933.07 - # 2016-01-09 17:20:00 933.07 - # 2016-01-09 17:30:00 933.07 - # 2016-01-09 17:40:00 932.07 - # 2016-01-09 17:50:00 932.07 - # 2016-01-09 18:00:00 932.07 - # dtype: float64 - - """ - - # check input types - for var, var_name in zip([ref_air_pressure_hPa, ref_air_temp_degC], ['ref_air_pressure_hPa', 'ref_air_temp_degC']): - if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series)): - raise TypeError(f"{var_name} must be a float or int or pandas.Series.") - for var, var_name in zip([ref_height_m, target_height_m], ['ref_height_m', 'target_height_m']): - if not (isinstance(var, float) or isinstance(var, int)): - raise TypeError(f"{var_name} must be a float or int.") - - # check dimensions of ref_air_pressure_hPa and ref_air_temp_degC if Series - if isinstance(ref_air_pressure_hPa, pd.Series) and (isinstance(ref_air_temp_degC, pd.Series)): - if len(ref_air_pressure_hPa) != len(ref_air_temp_degC): - raise ValueError("ref_air_pressure_hPa and ref_air_temp_degC must have the same dimensions.") - - # Constants as outlined in ISO:2533 - g = ACCEL_DUE_TO_GRAVITY # Acceleration due to gravity (m/s^2) - L = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE # Temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) - R = GAS_CONST_DRY_AIR # Specific gas constant dry air - - ref_air_temp_K = ref_air_temp_degC + 273.15 # Convert temp units to K - - scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) - ) ** (-g / (L * R))) - - return scaled_air_pressure_hPa - - -def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], - ref_height_m: float, - target_height_m: float, - lapse_rate_kg_m3_m: float = (0.001 * AIR_DENSITY_LAPSE_RATE) - ) -> Union[float, pd.Series]: - """ - Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height - (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. - - :param ref_air_density_kg_m3: Reference air density value(s) in kg/m3. - :type ref_air_density_kg_m3: float or pandas.Series - :param ref_height_m: Measurement height (in metres) of ref_air_density_kg_m3. - :type ref_height_m float - :param target_height_m: Height (in metres) that ref_air_density_kg_m3 is scaled to. - :type target_height_m: float - :param lapse_rate_kg_m3_m: Lapse rate describes how air density changes with increasing height above the - earth's surface in kg/m3/m. - Default value of -0.113 kg/m3 per km above earth's surface (-0.000113 kg/m3/m) - taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014). - :type lapse_rate_kg_m3_m: float - :return: Air density at specified height of target_height_m in kg/m3. Type depends on - type(ref_air_density_kg_m3) input. - :rtype: float or pandas.Series - - **Example usage** - :: - import brightwind as bw - - # scale float value of air density using default lapse_rate_kg_m3_m - bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) - # 1.22174 - - # scale float value of air density using non-default value for lapse_rate_kg_m3_m - bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100, - lapse_rate_kg_m3_m=-0.0002) - # 1.22 - - # derive air density and scale based on series input values for reference air density - data = bw.load_csv(bw.demo_datasets.demo_data) - test_density = bw.calc_air_density(data.T2m, data.P2m) - bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10) - - Timestamp - 2016-01-09 17:10:00 1.186780 - 2016-01-09 17:20:00 1.187175 - 2016-01-09 17:30:00 1.187747 - 2016-01-09 17:40:00 1.185950 - 2016-01-09 17:50:00 1.186301 - 2016-01-09 18:00:00 1.185686 - dtype: float64 - """ - - scaled_air_density = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, - y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) - - return scaled_air_density - - -def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series], - ref_height_m: float, - target_height_m: float, - lapse_rate_deg_m: float = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE - ) -> Union[float, pd.Series]: - """ - Linearly scales reference air temperature measurement (ref_air_temperature) from its measurement height - (ref_height_m) to the height specified as target_height_m, by applying the constant lapse_rate_deg_m. - - :param ref_air_temperature: Air temperature value(s) in degrees [for example in Celsius or Kelvin]. - :type ref_air_temperature: float or pandas.Series - :param ref_height_m: Measurement height (in metres) of ref_air_temperature. - :type ref_height_m: float - :param target_height_m: Height (in metres) that ref_air_temperature is scaled to. - :type target_height_m: float - :param lapse_rate_deg_m: Lapse rate describes how air temperature changes with increasing height - above the earth's surface. - Units should be degrees of temperature per unit of height, e.g. °C/m or K/m. - Default value of -6.5 degrees Celsius per km above the earth's surface - (or -0.0065 °C/m) is commonly used as an approximation of the - atmospheric lapse rate. - In particular, the IEC standards rely on the ISO2533:1975 Standard Atmosphere - which states that a lapse rate of 6.5 K/km is valid for geopotential altitudes - of up to 11 km above earth's surface. - This value was also adopted in WASP 11: - Mortensen, N. G., Heathfield, D. N., Rathmann, O., & Nielsen, M. (2014). - Wind Atlas Analysis and Application Program: WAsP 11 Help Facility. Computer - programme, Department of Wind Energy, Technical University of Denmark - https://orbit.dtu.dk/en/publications/wind-atlas-analysis-and-application-program-wasp-11-help-facility - :type lapse_rate_deg_m: float - :return: Air temperature at specified height of target_height_m in same unit as input - ref_air_temperature [for example in Celsius or Kelvin]. Output type depends on - type(ref_air_temperature) input. - :rtype: float or pandas.Series - - **Example usage** - :: - - import brightwind as bw - - # scale air temperature based on float input value for reference air temperature - bw.scale_air_temperature_to_height(ref_air_temperature=10.0065, ref_height_m=10, target_height_m=11) - # 10.0 - - # scale air temperature based on float input value for reference air temperature with non-default lapse_rate_deg_m - bw.scale_air_temperature_to_height(ref_air_temperature=10, ref_height_m=12, target_height_m=10, - lapse_rate_deg_m=-0.001) - # 10.002 - - # scale air temperature based on series input values for reference air temperature - data = bw.load_csv(bw.demo_datasets.demo_data) - - bw.scale_air_temperature_to_height(ref_air_temperature=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=20) - # Timestamp - # 2016-01-09 17:10:00 0.837 - # 2016-01-09 17:20:00 0.746 - # 2016-01-09 17:30:00 0.614 - # 2016-01-09 17:40:00 0.735 - # 2016-01-09 17:50:00 0.654 - # 2016-01-09 18:00:00 0.796 - # Name: T2m, dtype: float64 - """ - - scaled_air_temp = utils.linear_transform(x_target=target_height_m, x_ref=ref_height_m, - y_ref=ref_air_temperature, slope=lapse_rate_deg_m) - return scaled_air_temp diff --git a/brightwind/transform/scale.py b/brightwind/transform/scale.py new file mode 100644 index 00000000..7bf903ea --- /dev/null +++ b/brightwind/transform/scale.py @@ -0,0 +1,390 @@ +from typing import Union +import numpy as np +import pandas as pd + +__all__ = ['apply_scale_factor', + 'linear_transform', + 'scale_air_density_to_height', + 'scale_air_temperature_to_height', + 'scale_air_pressure_to_height'] + +# Acceleration due to gravity (m/s^2) from ISO:2533-1975 Standard Atmosphere +ACCEL_DUE_TO_GRAVITY = 9.80665 + +# Temperature lapse rate (K/m or degC/m) from ISO:2533-1975 Standard Atmosphere +TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE = -0.0065 + +# Specific gas constant for dry air (J/K/kg or m2/K/s2) from ISO:2533-1975 Standard Atmosphere +GAS_CONST_DRY_AIR = 287.05 + +# Air density lapse rate (kg/m3/km) from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014) +AIR_DENSITY_LAPSE_RATE = -0.113 + + +def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], + scale_factor: Union[float, int] + ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: + """ + Scales data by the scale_factor. + + If data input is pd.DataFrame, only numeric columns are scaled. + + :param data: Data value(s) to scale by the scale_factor. + :type data: float or int or pandas.Series or pandas.DataFrame or numpy.array + :param scale_factor: Scaling factor to use for scaling data values. + :type scale_factor: float or int + :returns: Scaled data value(s). Output type depends on type(data). + :rtype: float or int or pandas.Series or pandas.DataFrame or numpy.array + + **Example usage** + :: + import brightwind as bw + import pandas as pd + import numpy as np + + # scale float by scale_factor of 0.5 + bw.apply_scale_factor(3, 0.5) + # 1.5 + + # # scale np.array by scale_factor of 0.5 + bw.apply_scale_factor(np.array([0, 1, 2]), 0.5) + # array([0. , 0.5, 1. ]) + + # scale pd.Series by scale_factor of -10 + bw.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) + # 0 -100 + # 1 -200 + # 2 -300 + # 3 -400 + # dtype: int64 + + # scale pd.DataFrame by scale factor of 2 + df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) + bw.apply_scale_factor(df, 2) + # a b c + # 0 1.0 6 a + # 1 2.4 8 b + """ + + if not isinstance(data, (float, int, pd.DataFrame, pd.Series, np.ndarray)): + raise ValueError('data should be a float or int or pd.DataFrame or pd.Series or np.ndarray') + if not isinstance(scale_factor, (float, int)): + raise ValueError('scale_factor should be a float or int') + + if isinstance(data, pd.DataFrame): + # only apply scaling to numeric columns + numeric_df = scale_factor * (data.select_dtypes(include='number')) + result = pd.concat([numeric_df, data.select_dtypes(exclude='number')], axis=1) + return result + if isinstance(data, pd.Series): + if not pd.api.types.is_numeric_dtype(data): + raise ValueError('data inputted as a pandas.Series must be numeric') + if isinstance(data, np.ndarray): + if not np.issubdtype(data.dtype, np.number): + raise ValueError('data inputted as a np.ndarray must be numeric') + return scale_factor * data + + +def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], + x_ref: Union[float, int, np.ndarray, pd.Series], + y_ref: Union[float, int, np.ndarray, pd.Series], + slope: Union[float, int]) -> Union[float, int, np.ndarray, pd.Series]: + """ + Perform a linear transform of known (x_ref, y_ref) to calculate y_target, + using a constant slope and known value(s) of x_target. + + Function applies a linear transformation based on the equation of a straight line with constant slope: + y_target = slope * (x_target - x_ref) + y_ref, + where (x_ref, y_ref) is effectively considered a known point on a line with the inputted slope. + + Note that if pd.Series or np.ndarray inputs are provided for x_target, x_ref and y_ref, + then the transform is performed on a point by point basis, as in, the n^{th} value of + the resulting y_target would be: + y_target[n] = slope * (x_target[n] - x_ref[n]) + y_ref[n], + where (x_ref[n], y_ref[n]) is effectively considered a known point on a line with the inputted slope. + Therefore all three of x_target, x_ref and y_ref must have the same dimensions. + The inputted slope is used even in the case of pd.Series or np.ndarray inputs as the purpose of this function + is to perform a linear transformation using a predefined slope, not to fit a line to the input data. + + :param x_target: Target x value(s) at which to calculate y_target. + :type x_target: float or int or numpy.ndarray or pandas.Series + :param x_ref: Reference x value(s) of known reference point(s) on the line. + :type x_ref: float or int or numpy.ndarray or pandas.Series + :param y_ref: Reference y value(s) of known reference point(s) on the line. + :type y_ref: float or int or numpy.ndarray or pandas.Series + :param slope: Slope of the line equal to (y_target - y_ref) / (x_target - x_ref) where + ref and target x and y are any two points on the line. + :type slope: float or int + :return: Value(s) of y_target at specified x_target. + :rtype: float or int or numpy.ndarray or pandas.Series + + **Example usage** + :: + import brightwind as bw + + data = bw.load_csv(bw.demo_datasets.demo_data) + + # calculate y_target for x_target = 11 where (x_ref, y_ref) = (5, 10) is a point on the line and the slope is -0.5. + bw.linear_transform(x_target=11, x_ref=5, y_ref=10, slope=-0.5) + # 7.0 + + # calculate y_target for x_target as a pandas.Series and (x_ref, y_ref) = (2, 20) is a point on the line + # and the slope is -0.0065. + bw.linear_transform( + x_target=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065) + # Timestamp + # 2016-01-09 17:10:00 20.006799 + # 2016-01-09 17:20:00 20.007390 + # 2016-01-09 17:30:00 20.008249 + # 2016-01-09 17:40:00 20.007462 + # 2016-01-09 17:50:00 20.007988 + # 2016-01-09 18:00:00 20.007065 + # Name: T2m, dtype: float64 + + # calculate y_target for x_target and x_ref as a int and y_ref as a np.ndarray where (x_ref, y_ref) = (2, y_ref) + # are points on the line and the slope is -0.0065. + bw.linear_transform( + x_target=20, + x_ref=2, + y_ref=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) + # array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796]) + """ + # check input types + for var, var_name in zip([x_ref, y_ref, x_target], ['x_ref', 'y_ref', 'x_target']): + if not isinstance(var, (float, int, pd.Series, np.ndarray)): + raise TypeError(f"{var_name} must be a float or int or a numpy.ndarray or pandas.Series.") + + if not isinstance(slope, (float, int)): + raise TypeError("slope must be a float or int.") + + # check dimensions of x_ref and x_target if arrays or Series + if (isinstance(x_ref, (np.ndarray, pd.Series))) and ( + isinstance(x_target, (np.ndarray, pd.Series))): + if len(x_ref) != len(x_target): + raise ValueError("x_ref and x_target must have the same dimensions.") + + # check dimensions of y_ref if arrays or Series + if isinstance(x_target - x_ref, (np.ndarray, pd.Series)): + if isinstance(y_ref, (np.ndarray, pd.Series)): + if len(y_ref) != len(x_target - x_ref): + raise ValueError("y_ref must have the same dimensions as x_target or x_ref.") + + y_target = slope * (x_target - x_ref) + y_ref + + return y_target + + +def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_kg_m3_m: float = (0.001 * AIR_DENSITY_LAPSE_RATE) + ) -> Union[float, pd.Series]: + """ + Linearly scales reference air density measurement (ref_air_density_kg_m3) from its measurement height + (ref_height_m) to the height specified as the target_height_m, by applying a constant lapse_rate_kg_m3_m. + + :param ref_air_density_kg_m3: Reference air density value(s) in kg/m3. + :type ref_air_density_kg_m3: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_density_kg_m3. + :type ref_height_m float + :param target_height_m: Height (in metres) that ref_air_density_kg_m3 is scaled to. + :type target_height_m: float + :param lapse_rate_kg_m3_m: Lapse rate describes how air density changes with increasing height above the + earth's surface in kg/m3/m. + Default value of -0.113 kg/m3 per km above earth's surface (-0.000113 kg/m3/m) + taken from WindFarmer Theory Manual Version 5.3, DNV GL (April 2014). + :type lapse_rate_kg_m3_m: float + :return: Air density at specified height of target_height_m in kg/m3. Type depends on + type(ref_air_density_kg_m3) input. + :rtype: float or pandas.Series + + **Example usage** + :: + import brightwind as bw + + # scale float value of air density using default lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) + # 1.22174 + + # scale float value of air density using non-default value for lapse_rate_kg_m3_m + bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) + # 1.22 + + # derive air density and scale based on series input values for reference air density + data = bw.load_csv(bw.demo_datasets.demo_data) + test_density = bw.calc_air_density(data.T2m, data.P2m) + bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10) + + Timestamp + 2016-01-09 17:10:00 1.186780 + 2016-01-09 17:20:00 1.187175 + 2016-01-09 17:30:00 1.187747 + 2016-01-09 17:40:00 1.185950 + 2016-01-09 17:50:00 1.186301 + 2016-01-09 18:00:00 1.185686 + dtype: float64 + """ + + scaled_air_density = linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) + + return scaled_air_density + + +def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series], + ref_height_m: float, + target_height_m: float, + lapse_rate_deg_m: float = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE + ) -> Union[float, pd.Series]: + """ + Linearly scales reference air temperature measurement (ref_air_temperature) from its measurement height + (ref_height_m) to the height specified as target_height_m, by applying the constant lapse_rate_deg_m. + + :param ref_air_temperature: Air temperature value(s) in degrees [for example in Celsius or Kelvin]. + :type ref_air_temperature: float or pandas.Series + :param ref_height_m: Measurement height (in metres) of ref_air_temperature. + :type ref_height_m: float + :param target_height_m: Height (in metres) that ref_air_temperature is scaled to. + :type target_height_m: float + :param lapse_rate_deg_m: Lapse rate describes how air temperature changes with increasing height + above the earth's surface. + Units should be degrees of temperature per unit of height, e.g. °C/m or K/m. + Default value of -6.5 degrees Celsius per km above the earth's surface + (or -0.0065 °C/m) is commonly used as an approximation of the + atmospheric lapse rate. + In particular, the IEC standards rely on the ISO2533:1975 Standard Atmosphere + which states that a lapse rate of 6.5 K/km is valid for geopotential altitudes + of up to 11 km above earth's surface. + This value was also adopted in WASP 11: + Mortensen, N. G., Heathfield, D. N., Rathmann, O., & Nielsen, M. (2014). + Wind Atlas Analysis and Application Program: WAsP 11 Help Facility. Computer + programme, Department of Wind Energy, Technical University of Denmark + https://orbit.dtu.dk/en/publications/wind-atlas-analysis-and-application-program-wasp-11-help-facility + :type lapse_rate_deg_m: float + :return: Air temperature at specified height of target_height_m in same unit as input + ref_air_temperature [for example in Celsius or Kelvin]. Output type depends on + type(ref_air_temperature) input. + :rtype: float or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale air temperature based on float input value for reference air temperature + bw.scale_air_temperature_to_height(ref_air_temperature=10.0065, ref_height_m=10, target_height_m=11) + # 10.0 + + # scale air temperature based on float input value for reference air temperature with non-default lapse_rate_deg_m + bw.scale_air_temperature_to_height(ref_air_temperature=10, ref_height_m=12, target_height_m=10, + lapse_rate_deg_m=-0.001) + # 10.002 + + # scale air temperature based on series input values for reference air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_temperature_to_height(ref_air_temperature=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=20) + # Timestamp + # 2016-01-09 17:10:00 0.837 + # 2016-01-09 17:20:00 0.746 + # 2016-01-09 17:30:00 0.614 + # 2016-01-09 17:40:00 0.735 + # 2016-01-09 17:50:00 0.654 + # 2016-01-09 18:00:00 0.796 + # Name: T2m, dtype: float64 + """ + + scaled_air_temp = linear_transform(x_target=target_height_m, x_ref=ref_height_m, + y_ref=ref_air_temperature, slope=lapse_rate_deg_m) + return scaled_air_temp + + +def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], + ref_air_temp_degC: Union[float, pd.Series], + ref_height_m: Union[float, int], + target_height_m: Union[float, int] + ) -> Union[float, int, pd.Series]: + """ + Calculates air pressure at target height (target_height_m) using reference air pressure (ref_air_pressure_hPa) + and air temperature (ref_air_temp_degC) values at a reference height (ref_height_m). + + Calculation based on ISO:2533-1975 Standard Atmosphere (https://www.iso.org/obp/ui/#iso:std:iso:2533:en) + as suggested by IEC 61400-12-1: + + scaled_air_pressure = ref_air_pressure_hPa*((1 + (L/ref_air_temp_K)*(target_height_m - ref_height_m))**(-g/(L*R))) + where: + g = 9.80665 is acceleration due to gravity (m/s^2) + L = -0.0065 is the temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = 287.05 is the specific gas constant for dry air (J/K/kg or m2/K/s2) + + :param ref_air_pressure_hPa: Reference air pressure value(s) in hPa (1mbar = 1hPa = 100Pa). + :type ref_air_pressure_hPa: float or int or pandas.Series + :param ref_air_temp_degC: Reference air temperature value(s) in degrees celsius. + :type ref_air_temp_degC: float or int or pandas.Series + :param ref_height_m: Height (in metres) of reference air temperature (ref_air_temp_degC) + and air pressure (ref_air_pressure_hPa). + :type ref_height_m: float or int + :param target_height_m: Height (in metres) which ref_air_pressure_hPa is scaled to. + :type target_height_m float or int + :return: Air pressure at specified height of target_height_m in hPa (1mbar = 1hPa = 100Pa). + Type depends on type(ref_air_pressure_hPa) and type(ref_air_temp_degC) inputs. + :rtype: float or int or pandas.Series + + **Example usage** + :: + + import brightwind as bw + + # scale float value of air pressure + round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, ref_height_m=10, + target_height_m=200), 2) + # 977.45 + + # scale air pressure based on input series of reference air pressure and air temperature + data = bw.load_csv(bw.demo_datasets.demo_data) + + bw.scale_air_pressure_to_height(ref_air_pressure_hPa=data['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=data['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10).round(2) + # Timestamp + # 2016-01-09 17:10:00 933.07 + # 2016-01-09 17:20:00 933.07 + # 2016-01-09 17:30:00 933.07 + # 2016-01-09 17:40:00 932.07 + # 2016-01-09 17:50:00 932.07 + # 2016-01-09 18:00:00 932.07 + # dtype: float64 + + """ + + # check input types + for var, var_name in zip([ref_air_pressure_hPa, ref_air_temp_degC], ['ref_air_pressure_hPa', 'ref_air_temp_degC']): + if not isinstance(var, (float, int, pd.Series)): + raise TypeError(f"{var_name} must be a float or int or pandas.Series.") + for var, var_name in zip([ref_height_m, target_height_m], ['ref_height_m', 'target_height_m']): + if not isinstance(var, (float, int)): + raise TypeError(f"{var_name} must be a float or int.") + + # check dimensions of ref_air_pressure_hPa and ref_air_temp_degC if Series + if isinstance(ref_air_pressure_hPa, pd.Series) and (isinstance(ref_air_temp_degC, pd.Series)): + if len(ref_air_pressure_hPa) != len(ref_air_temp_degC): + raise ValueError("ref_air_pressure_hPa and ref_air_temp_degC must have the same dimensions.") + + # Constants as outlined in ISO:2533 + g = ACCEL_DUE_TO_GRAVITY # Acceleration due to gravity (m/s^2) + L = TEMP_LAPSE_RATE_STANDARD_ATMOSPHERE # Temperature lapse rate (K/m) (denoted beta in ISO:2533 notation) + R = GAS_CONST_DRY_AIR # Specific gas constant dry air + + ref_air_temp_K = ref_air_temp_degC + 273.15 # Convert temp units to K + + scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) + ) ** (-g / (L * R))) + + return scaled_air_pressure_hPa diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index edcf116c..509f687c 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +import brightwind.transform.scale from brightwind.utils import utils from brightwind.load.station import _Measurements from brightwind.load.station import DATE_INSTEAD_OF_NONE @@ -1043,7 +1044,7 @@ def scale_wind_speed(spd: Union[pd.Series, pd.DataFrame, float, int], # 2017-11-23 10:40:00 14.318 13.336 # 2017-11-23 10:50:00 12.808 11.498 """ - return utils.apply_scale_factor(spd, scale_factor) + return brightwind.transform.scale.apply_scale_factor(spd, scale_factor) def offset_wind_direction(wdir, offset: float): diff --git a/brightwind/utils/utils.py b/brightwind/utils/utils.py index 83adf1c6..2c56188a 100644 --- a/brightwind/utils/utils.py +++ b/brightwind/utils/utils.py @@ -9,9 +9,8 @@ 'validate_coverage_threshold', 'is_file', 'is_file_extension', - 'validate_json', - 'linear_transform', - 'apply_scale_factor'] + 'validate_json' + ] def _range_0_to_360(direction): @@ -257,152 +256,5 @@ def validate_json(json_to_check, schema): return data_is_valid -def linear_transform(x_target: Union[float, int, np.ndarray, pd.Series], - x_ref: Union[float, int, np.ndarray, pd.Series], - y_ref: Union[float, int, np.ndarray, pd.Series], - slope: Union[float, int]) -> Union[float, int, np.ndarray, pd.Series]: - """ - Perform a linear transform of known (x_ref, y_ref) to calculate y_target, - using a constant slope and known value(s) of x_target. - - Function applies a linear transformation based on the equation of a straight line with constant slope: - y_target = slope * (x_target - x_ref) + y_ref, - where (x_ref, y_ref) is effectively considered a known point on a line with the inputted slope. - - Note that if pd.Series or np.ndarray inputs are provided for x_target, x_ref and y_ref, - then the transform is performed on a point by point basis, as in, the n^{th} value of - the resulting y_target would be: - y_target[n] = slope * (x_target[n] - x_ref[n]) + y_ref[n], - where (x_ref[n], y_ref[n]) is effectively considered a known point on a line with the inputted slope. - Therefore all three of x_target, x_ref and y_ref must have the same dimensions. - The inputted slope is used even in the case of pd.Series or np.ndarray inputs as the purpose of this function - is to perform a linear transformation using a predefined slope, not to fit a line to the input data. - - :param x_target: Target x value(s) at which to calculate y_target. - :type x_target: float or int or numpy.ndarray or pandas.Series - :param x_ref: Reference x value(s) of known reference point(s) on the line. - :type x_ref: float or int or numpy.ndarray or pandas.Series - :param y_ref: Reference y value(s) of known reference point(s) on the line. - :type y_ref: float or int or numpy.ndarray or pandas.Series - :param slope: Slope of the line equal to (y_target - y_ref) / (x_target - x_ref) where - ref and target x and y are any two points on the line. - :type slope: float or int - :return: Value(s) of y_target at specified x_target. - :rtype: float or int or numpy.ndarray or pandas.Series - - **Example usage** - :: - import brightwind as bw - - data = bw.load_csv(bw.demo_datasets.demo_data) - - # calculate y_target for x_target = 11 where (x_ref, y_ref) = (5, 10) is a point on the line and the slope is -0.5. - bw.utils.utils.linear_transform(x_target=11, x_ref=5, y_ref=10, slope=-0.5) - # 7.0 - - # calculate y_target for x_target as a pandas.Series and (x_ref, y_ref) = (2, 20) is a point on the line - # and the slope is -0.0065. - bw.utils.utils.linear_transform( - x_target=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - x_ref=2, - y_ref=20, - slope=-0.0065) - - # calculate y_target for x_target and x_ref as a int and y_ref as a np.ndarray where (x_ref, y_ref) = (2, y_ref) - # are points on the line and the slope is -0.0065. - bw.utils.utils.linear_transform( - x_target=20, - x_ref=2, - y_ref=data.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, - slope=-0.0065) - """ - # check input types - for var, var_name in zip([x_ref, y_ref, x_target], ['x_ref', 'y_ref', 'x_target']): - if not (isinstance(var, float) or isinstance(var, int) or isinstance(var, pd.Series) or - isinstance(var, np.ndarray)): - raise TypeError(f"{var_name} must be a float or int or a numpy.ndarray or pandas.Series.") - - if not (isinstance(slope, float) or isinstance(slope, int)): - raise TypeError("slope must be a float or int.") - - # check dimensions of x_ref and x_target if arrays or Series - if (isinstance(x_ref, np.ndarray) or isinstance(x_ref, pd.Series)) and ( - isinstance(x_target, np.ndarray) or isinstance(x_target, pd.Series)): - if len(x_ref) != len(x_target): - raise ValueError("x_ref and x_target must have the same dimensions.") - - # check dimensions of y_ref if arrays or Series - if isinstance(x_target - x_ref, np.ndarray) or isinstance(x_target - x_ref, pd.Series): - if isinstance(y_ref, np.ndarray) or isinstance(y_ref, pd.Series): - if len(y_ref) != len(x_target - x_ref): - raise ValueError("y_ref must have the same dimensions as x_target or x_ref.") - - y_target = slope * (x_target - x_ref) + y_ref - - return y_target -def apply_scale_factor(data: Union[float, int, pd.DataFrame, pd.Series, np.array], - scale_factor: Union[float, int] - ) -> Union[float, int, pd.DataFrame, pd.Series, np.array]: - """ - Scales data by the scale_factor. - - If data input is pd.DataFrame, only numeric columns are scaled. - - :param data: Data value(s) to scale by the scale_factor. - :type data: float or int or pandas.Series or pandas.DataFrame or numpy.array - :param scale_factor: Scaling factor to use for scaling data values. - :type scale_factor: float or int - :returns: Scaled data value(s). Output type depends on type(data). - :rtype: float or int or pandas.Series or pandas.DataFrame or numpy.array - - **Example usage** - :: - import brightwind as bw - import pandas as pd - import numpy as np - - # scale float by scale_factor of 0.5 - bw.utils.utils.apply_scale_factor(3, 0.5) - # 1.5 - - # # scale np.array by scale_factor of 0.5 - # bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) - # array([0. , 0.5, 1. ]) - - # scale pd.Series by scale_factor of -10 - bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) - # 0 -100 - # 1 -200 - # 2 -300 - # 3 -400 - # dtype: int64 - - # scale pd.DataFrame by scale factor of 2 - df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) - bw.utils.utils.apply_scale_factor(df, 2) - # a b c - # 0 1.0 6 a - # 1 2.4 8 b - """ - - if not isinstance(data, (float, int, pd.DataFrame, pd.Series, np.ndarray)): - raise ValueError('data should be a float or int or pd.DataFrame or pd.Series or np.ndarray') - if not isinstance(scale_factor, (float, int)): - raise ValueError('scale_factor should be a float or int') - - if isinstance(data, pd.DataFrame): - # only apply scaling to numeric columns - numeric_df = scale_factor * (data.select_dtypes(include='number')) - result = pd.concat([numeric_df, data.select_dtypes(exclude='number')], axis=1) - return result - else: - if isinstance(data, pd.Series): - if not pd.api.types.is_numeric_dtype(data): - raise ValueError('data inputted as a pandas.Series must be numeric') - if isinstance(data, np.ndarray): - if not np.issubdtype(data.dtype, np.number): - raise ValueError('data inputted as a np.ndarray must be numeric') - - return scale_factor * data diff --git a/tests/test_analyse.py b/tests/test_analyse.py index a376c297..2a3cf01a 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -667,7 +667,7 @@ def test_calc_air_density(): # test float/int inputs with elevation adjustment assert bw.calc_air_density(15, 1013, elevation_ref=0, elevation_site=200) == 1.203 assert bw.calc_air_density(15, 1013, rel_humidity_percent=50, elevation_ref=0, elevation_site=200 - ) - bw.scale_air_density_to_height( + ) - bw.transform.scale.scale_air_density_to_height( bw.calc_air_density(15, 1013, rel_humidity_percent=50), 0, 200) <1e-3 # test errors @@ -695,87 +695,3 @@ def test_dist_matrix_by_direction_sector(): bw.dist_matrix_by_dir_sector(DATA.Spd40mN, DATA.T2m, DATA.Dir38mS, var_to_bin_by_array=[-8, -5, 5, 10, 15, 20, 26], sectors=8) assert True - - -def test_scale_air_pressure_to_height(): - assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, - ref_height_m=10, target_height_m=200), 2)== 977.45 - assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, - ref_height_m=10, target_height_m=15), 2) == 999.4 - pd.testing.assert_series_equal(bw.scale_air_pressure_to_height( - ref_air_pressure_hPa=DATA['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_air_temp_degC=DATA['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10), - pd.Series(data = [933.07, 933.07, 933.07, 932.07, 932.07, 932.07], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - # test error raising for invalid input types - with pytest.raises(TypeError): - bw.scale_air_pressure_to_height( - ref_air_pressure_hPa='invalid_type', - ref_air_temp_degC=12, - ref_height_m=2, - target_height_m=10) - - # test error raising for invalid slope type - with pytest.raises(TypeError): - bw.scale_air_pressure_to_height( - ref_air_pressure_hPa=1000, - ref_air_temp_degC=12, - ref_height_m=2, - target_height_m='invalid_type') - - # test error raising for mismatched dimensions with Series - with pytest.raises(ValueError): - bw.scale_air_pressure_to_height( - ref_air_pressure_hPa=pd.Series([1, 2, 3]), - ref_air_temp_degC=pd.Series([1, 2]), - ref_height_m=2, - target_height_m=10) - -def test_scale_air_density_to_height(): - assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) == 1.22174 - - assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, - ref_height_m=80, - target_height_m=100, - lapse_rate_kg_m3_m=-0.0002) == 1.22 - - test_density = bw.calc_air_density(DATA.T2m, DATA.P2m) - pd.testing.assert_series_equal(bw.scale_air_density_to_height( - ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, target_height_m=10), - pd.Series(data = [1.186780, 1.187175, 1.187747, 1.185950, 1.186301 , 1.185686], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - -def test_scale_air_temperature_to_height(): - assert bw.scale_air_temperature_to_height(10.0065, 10, 11) == 10.0 - - assert bw.scale_air_temperature_to_height(ref_air_temperature=10, - ref_height_m=12, - target_height_m=10, - lapse_rate_deg_m=-0.001) == 10.002 - - pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( - ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, - target_height_m=20, - lapse_rate_deg_m=-0.001), - pd.Series(data=[0.936, 0.845, 0.713, 0.834, 0.753, 0.895], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - pd.testing.assert_series_equal(bw.scale_air_temperature_to_height( - ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - ref_height_m=2, - target_height_m=20), - pd.Series(data=[0.837, 0.746, 0.614, 0.735, 0.654, 0.796], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) diff --git a/tests/test_scale.py b/tests/test_scale.py new file mode 100644 index 00000000..f63d66dc --- /dev/null +++ b/tests/test_scale.py @@ -0,0 +1,160 @@ +import pytest +import brightwind as bw +import pandas as pd +import numpy as np + +DATA = bw.load_csv(bw.demo_datasets.demo_data) +DATA = bw.apply_cleaning(DATA, bw.demo_datasets.demo_cleaning_file) + + +def test_scale_air_pressure_to_height(): + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=200), 2) == 977.45 + assert round(bw.scale_air_pressure_to_height(ref_air_pressure_hPa=1000, ref_air_temp_degC=12, + ref_height_m=10, target_height_m=15), 2) == 999.4 + pd.testing.assert_series_equal(bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=DATA['P2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_air_temp_degC=DATA['T2m'].loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data=[933.07, 933.07, 933.07, 932.07, 932.07, 932.07], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + # test error raising for invalid input types + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa='invalid_type', + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m=10) + + # test error raising for invalid slope type + with pytest.raises(TypeError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=1000, + ref_air_temp_degC=12, + ref_height_m=2, + target_height_m='invalid_type') + + # test error raising for mismatched dimensions with Series + with pytest.raises(ValueError): + bw.scale_air_pressure_to_height( + ref_air_pressure_hPa=pd.Series([1, 2, 3]), + ref_air_temp_degC=pd.Series([1, 2]), + ref_height_m=2, + target_height_m=10) + + +def test_scale_air_density_to_height(): + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, ref_height_m=80, target_height_m=100) == 1.22174 + + assert bw.scale_air_density_to_height(ref_air_density_kg_m3=1.224, + ref_height_m=80, + target_height_m=100, + lapse_rate_kg_m3_m=-0.0002) == 1.22 + + test_density = bw.calc_air_density(DATA.T2m, DATA.P2m) + pd.testing.assert_series_equal(bw.scale_air_density_to_height( + ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, target_height_m=10), + pd.Series(data=[1.186780, 1.187175, 1.187747, 1.185950, 1.186301, 1.185686], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + +def test_scale_air_temperature_to_height(): + assert bw.scale_air_temperature_to_height(10.0065, 10, 11) == 10.0 + + assert bw.scale_air_temperature_to_height(ref_air_temperature=10, + ref_height_m=12, + target_height_m=10, + lapse_rate_deg_m=-0.001) == 10.002 + + pd.testing.assert_series_equal( + bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20, + lapse_rate_deg_m=-0.001), + pd.Series(data=[0.936, 0.845, 0.713, 0.834, 0.753, 0.895], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + pd.testing.assert_series_equal( + bw.scale_air_temperature_to_height( + ref_air_temperature=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + ref_height_m=2, + target_height_m=20), + pd.Series(data=[0.837, 0.746, 0.614, 0.735, 0.654, 0.796], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + + +def test_linear_transform(): + # test with float and int inputs + assert bw.linear_transform(x_target=10, x_ref=5, y_ref=10, slope=-0.5) == 7.5 + + +# test with array input for y_ref +assert (bw.linear_transform( + x_target=20, + x_ref=2, + y_ref=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, + slope=-0.0065) == np.array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796])).all() + +# test with pandas Series input for y_ref +pd.testing.assert_series_equal( + bw.linear_transform( + x_target=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], + x_ref=2, + y_ref=20, + slope=-0.0065).round(3), + pd.Series(data=[20.007, 20.007, 20.008, 20.007, 20.008, 20.007], + index=pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', + '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), + check_names=False) + +# test error raising for invalid input types +with pytest.raises(TypeError): + bw.linear_transform( + x_target='invalid_type', + x_ref=2, + y_ref=20, + slope=-0.0065) + +# test error raising for invalid slope type +with pytest.raises(TypeError): + bw.linear_transform( + x_target=10, + x_ref=2, + y_ref=20, + slope='invalid_type') + +# test error raising for mismatched dimensions +with pytest.raises(ValueError): + bw.linear_transform( + x_target=np.array([1, 2, 3]), + x_ref=np.array([1, 2]), + y_ref=20, + slope=-0.5) + +# test error raising for mismatched dimensions with Series +with pytest.raises(ValueError): + bw.linear_transform( + x_target=pd.Series([1, 2, 3]), + x_ref=2, + y_ref=pd.Series([1, 2]), + slope=-0.5) + + +def test_apply_scale_factor(): + assert bw.apply_scale_factor(3, 0.5) == 1.5 + assert (bw.apply_scale_factor(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() + assert (bw.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) == [-100, -200, -300, -400]).all() + df = pd.DataFrame({'a': [0.5, 1.2], 'b': [3, 4], 'c': ['a', 'b']}) + result_df = pd.DataFrame({'a': [1.0, 2.4], 'b': [6, 8], 'c': ['a', 'b']}) + assert result_df.equals(bw.apply_scale_factor(df, 2)) diff --git a/tests/test_transform.py b/tests/test_transform.py index 90a37fec..3a1c6ebb 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -804,8 +804,8 @@ def test_scale_wind_speed(): assert bw.scale_wind_speed(3, 0.5) == 1.5 assert (bw.scale_wind_speed(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() - assert (bw.scale_wind_speed(DATA_CLND['Spd40mN'].tail(3), 0.5)==[4.015 , 3.4055, 2.9325]).all() - - df = pd.DataFrame({'Spd_100m':[0.5, 1.2], 'Spd_100m':[3, 4], 'c':['a', 'b']}) - result_df = pd.DataFrame({'Spd_100m':[1.0, 2.4], 'Spd_100m':[6, 8], 'c':['a', 'b']}) + assert (bw.scale_wind_speed(DATA_CLND['Spd40mN'].tail(3), 0.5) == [4.015, 3.4055, 2.9325]).all() + + df = pd.DataFrame({'Spd_100m': [0.5, 1.2], 'Spd_101m': [3, 4], 'c': ['a', 'b']}) + result_df = pd.DataFrame({'Spd_100m': [1.0, 2.4], 'Spd_101m': [6, 8], 'c': ['a', 'b']}) assert result_df.equals(bw.scale_wind_speed(df, 2)) diff --git a/tests/test_utils.py b/tests/test_utils.py index a188573f..e48096b0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -26,66 +26,3 @@ def test_slice_data(): data_sliced = bw.utils.utils.slice_data(DATA, date_to='2017-10-23') assert data_sliced.index[0] == DATA.index[0] - - -def test_linear_transform(): - # test with float and int inputs - assert bw.utils.utils.linear_transform(x_target=10, x_ref=5, y_ref=10, slope=-0.5) == 7.5 - - # test with array input for y_ref - assert (bw.utils.utils.linear_transform( - x_target=20, - x_ref=2, - y_ref=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'].values, - slope=-0.0065) == np.array([0.837, 0.746, 0.614, 0.735, 0.654, 0.796])).all() - - # test with pandas Series input for y_ref - pd.testing.assert_series_equal(bw.utils.utils.linear_transform( - x_target=DATA.T2m.loc['2016-01-09 17:10':'2016-01-09 18:00'], - x_ref=2, - y_ref=20, - slope=-0.0065).round(3), - pd.Series(data = [20.007, 20.007, 20.008, 20.007, 20.008, 20.007], - index = pd.to_datetime(['2016-01-09 17:10:00', '2016-01-09 17:20:00', '2016-01-09 17:30:00', - '2016-01-09 17:40:00', '2016-01-09 17:50:00', '2016-01-09 18:00:00'])), - check_names=False) - - # test error raising for invalid input types - with pytest.raises(TypeError): - bw.utils.utils.linear_transform( - x_target='invalid_type', - x_ref=2, - y_ref=20, - slope=-0.0065) - - #test error raising for invalid slope type - with pytest.raises(TypeError): - bw.utils.utils.linear_transform( - x_target=10, - x_ref=2, - y_ref=20, - slope='invalid_type') - - #test error raising for mismatched dimensions - with pytest.raises(ValueError): - bw.utils.utils.linear_transform( - x_target=np.array([1, 2, 3]), - x_ref=np.array([1, 2]), - y_ref=20, - slope=-0.5) - - #test error raising for mismatched dimensions with Series - with pytest.raises(ValueError): - bw.utils.utils.linear_transform( - x_target=pd.Series([1, 2, 3]), - x_ref=2, - y_ref=pd.Series([1, 2]), - slope=-0.5) - -def test_apply_scale_factor(): - assert bw.utils.utils.apply_scale_factor(3, 0.5) == 1.5 - assert (bw.utils.utils.apply_scale_factor(np.array([0, 1, 2]), 0.5) == [0, 0.5, 1]).all() - assert (bw.utils.utils.apply_scale_factor(pd.Series([10, 20, 30, 40]), -10) == [-100, -200, -300, -400]).all() - df = pd.DataFrame({'a':[0.5, 1.2], 'b':[3, 4], 'c':['a', 'b']}) - result_df = pd.DataFrame({'a':[1.0, 2.4], 'b':[6, 8], 'c':['a', 'b']}) - assert result_df.equals(bw.utils.utils.apply_scale_factor(df, 2)) From 88e3c21ee73016c48f675e86001b718b59350951 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Fri, 28 Nov 2025 10:35:39 +0000 Subject: [PATCH 17/27] iss #520 return deadband offset results table (#551) * iss #520 return deadband offset table * iss #520 adding to changelog * iss #520 update to include height * iss #520 minor formatting * iss #520 move location in changelog * iss #520 response to comments * iss #520 improved docstring example --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 1 + brightwind/transform/transform.py | 56 ++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55720797..7efae53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Additional labels for pre-release and build metadata are available as extensions 3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) 4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) 5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) +1. Added optional output to `apply_wind_vane_deadband_offset()`, which provides a table of deadband offset, logger offset and applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) ## [2.3.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index 509f687c..c51ce8c1 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1063,7 +1063,7 @@ def offset_wind_direction(wdir, offset: float): return wdir.add(offset).apply(utils._range_0_to_360) -def apply_wind_vane_deadband_offset(data, measurements, inplace=False): +def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_results_table=False): """ Automatically apply deadband offsets of the wind vanes to the timeseries data. The deadband orientation information for each wind direction measurement and time period is contained in the measurements @@ -1082,18 +1082,23 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): This function accounts for this adjustment. - :param data: Timeseries data. - :type data: pd.DataFrame or pd.Series - :param measurements: Measurement information extracted from a WRA Data Model using bw.MeasurementStation - :type measurements: list or dict or _Measurements - :param inplace: If 'inplace' is True, the original direction data, contained in 'data', will be - modified and replaced with the adjusted direction data. If 'inplace' is False, the - original data will not be touched and instead a new DataFrame containing the adjusted - direction data is created. To store this adjusted direction data, please ensure it is - assigned to a new variable. - :type inplace: bool - :return: Data with adjusted wind direction by the deadband orientation. - :rtype: pd.DataFrame or pd.Series + :param data: Timeseries data. + :type data: pd.DataFrame or pd.Series + :param measurements: Measurement information extracted from a WRA Data Model using bw.MeasurementStation + :type measurements: list or dict or _Measurements + :param inplace: If 'inplace' is True, the original direction data, contained in 'data', will be + modified and replaced with the adjusted direction data. If 'inplace' is False, the + original data will not be touched and instead a new DataFrame containing the + adjusted direction data is created. To store this adjusted direction data, please + ensure it is assigned to a new variable. + :type inplace: bool + :param return_results_table: Optional key to return a dataframe containing deadband offset, logger offset and + applied offset for each directional sensor and the time period it is relevant for. + :type return_results_table: pd.DataFrame + :return: Data with adjusted wind direction by the deadband orientation, or where + return_results_table is specified, a tuple of the data and a DataFrame of + deadband offsets. + :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] **Example usage** :: @@ -1121,6 +1126,12 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): bw.apply_wind_vane_deadband_offset(data['Dir78mS'], mm1.measurements['Dir78mS'], inplace=True) print('\nWind vane deadband offset adjustment is completed.') + Specifying that the deadband offsets should be returned:: + data_adj, results_table = bw.apply_wind_vane_deadband_offset( + data['Dir78mS'], mm1.measurements['Dir78mS'], inplace=True, return_results_table=True + ) + results_table + """ # Depending on what is sent, get wdir properties into a list of properties wdirs_properties = _get_consistent_properties_format(measurements, 'wind_direction') @@ -1133,6 +1144,7 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): df = pd.DataFrame(data) if type(data) == pd.Series else data # Apply the offset + rows = [] for wdir_prop in wdirs_properties: name = wdir_prop['name'] if name in df.columns: @@ -1167,6 +1179,16 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): else: print('{} has dead_band_orientation of None from {} to {}.\n' .format(utils.bold(name), utils.bold(date_from), utils.bold(date_to_txt))) + height = wdir_prop.get('height_m') + rows.append({ + "name": name, + "height": height, + "deadband_offset": deadband, + "logger_offset": logger_offset, + "offset_applied": offset, + "date_from": date_from, + "date_to": date_to + }) else: print('{} is not found in data.\n'.format(utils.bold(name))) @@ -1175,6 +1197,14 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False): # if a Series is sent, send back a Series if type(data) == pd.Series: df = df[df.columns[0]] + if return_results_table: + results_table = pd.DataFrame(rows).sort_values(by=["name", "date_from"]).groupby( + ["name", "height", "deadband_offset", "logger_offset", "offset_applied"] + ).agg({ + "date_from": "first", + "date_to": lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(level=[1, 2, 3, 4]) + return df, results_table return df From e8ae2ff18fd8b36f6f7443b639883041fd2aae27 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Fri, 28 Nov 2025 14:21:55 +0000 Subject: [PATCH 18/27] Iss520 rename table headers (#559) * iss #520 rename columns of output table * iss #520 removal of potentially problematic logic * iss #520 revised sorting logic * iss #520 improved logic for grouping neighbouring rows * iss #520 revised filtering and column name capitalisation --- brightwind/transform/transform.py | 45 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index c51ce8c1..eb4b2bd2 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1181,13 +1181,13 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re .format(utils.bold(name), utils.bold(date_from), utils.bold(date_to_txt))) height = wdir_prop.get('height_m') rows.append({ - "name": name, - "height": height, - "deadband_offset": deadband, - "logger_offset": logger_offset, - "offset_applied": offset, - "date_from": date_from, - "date_to": date_to + "Name": name, + "Height [m]": height, + "Vane Dead Band Orientation [deg]": deadband, + "Logger Offset": logger_offset, + "Offset Applied [deg]": offset, + "Date From": date_from, + "Date To": date_to }) else: print('{} is not found in data.\n'.format(utils.bold(name))) @@ -1198,12 +1198,31 @@ def apply_wind_vane_deadband_offset(data, measurements, inplace=False, return_re if type(data) == pd.Series: df = df[df.columns[0]] if return_results_table: - results_table = pd.DataFrame(rows).sort_values(by=["name", "date_from"]).groupby( - ["name", "height", "deadband_offset", "logger_offset", "offset_applied"] - ).agg({ - "date_from": "first", - "date_to": lambda x: None if any(d is None for d in x) else max(x) - }).reset_index(level=[1, 2, 3, 4]) + results_df = pd.DataFrame(rows).sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + results_df['consecutive_group'] = ( + (results_df['Name'] != results_df['Name'].shift()) | + (results_df['Height [m]'] != results_df['Height [m]'].shift()) | + (results_df['Vane Dead Band Orientation [deg]'] != results_df['Vane Dead Band Orientation [deg]'].shift()) | + (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | + (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) + ).cumsum() + + # Group and aggregate consecutive periods + results_table = results_df.groupby([ + 'Name', + 'consecutive_group', + 'Height [m]', + 'Vane Dead Band Orientation [deg]', + 'Logger Offset', + 'Offset Applied [deg]' + ]).agg({ + 'Date From': 'first', + 'Date To': lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(drop=False).drop(columns=['consecutive_group']).set_index("Name").sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) return df, results_table return df From e9eb5e07bf71c7d1e446a5e7c09558ba6b705eb9 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Fri, 28 Nov 2025 14:32:13 +0000 Subject: [PATCH 19/27] Iss521 return device orientation offset table (#552) * iss #521 added optional output of device orientation table * iss #521 rename parameter for return_direction_offset_summary * iss #521 changes relating to comments on similar PRs * iss #521 groupby logic removed as not correct * iss #521 updated to take names from JSON schema * iss #521 change Name of measurement point * iss #521 revised sorting logic and added height as column * iss #521 update changelog * iss #521 revised grouping of rows logic * iss #521 formatting updates and docstring revision * iss #521 capitalisation of Applied * iss #521 fix PEP8 and expanded comment --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 4 +- brightwind/transform/transform.py | 79 ++++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7efae53b..1e3b0536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.4.0-dev] 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) + ### Bug Fixes 1. @@ -23,7 +24,8 @@ Additional labels for pre-release and build metadata are available as extensions 3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) 4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) 5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) -1. Added optional output to `apply_wind_vane_deadband_offset()`, which provides a table of deadband offset, logger offset and applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) +1. Added optional output to `apply_wind_vane_deadband_offset()` which provides a results table showing the applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) +1. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). ## [2.3.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index eb4b2bd2..b123aaa9 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1492,7 +1492,9 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals return df_copy.sort_index() -def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inplace=False): +def apply_device_orientation_offset( + data, measurement_station, wdir_cols=[], inplace=False, return_results_table=False + ): """ Applies a device orientation offset to wind direction data from remote sensing devices (lidar, sodar, or floating lidar) to align measurements with north. @@ -1522,18 +1524,21 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp Overlapping periods with non-zero `device_orientation_deg` values in `vertical_profiler_properties` are not supported and will raise an error. - :param data: Timeseries data. - :type data: pd.DataFrame or pd.Series - :param measurement_station: A simplified object to represent the IEA Wind Task 43 WRA Data Model. - :type measurement_station: bw.MeasurementStation - :param wdir_cols: Wind direction column names to apply the offset to. If empty, all wind direction - columns in the data are used. Default is an empty list. - :type wdir_cols: list - :param inplace: If True, modifies `data` in place. If False, returns a new DataFrame/Series - with adjusted values. Default is False. - :type inplace: bool, optional - :return: Data with wind direction adjusted by the orientation offset. - :rtype: pd.DataFrame or pd.Series + :param data: Timeseries data. + :type data: pd.DataFrame or pd.Series + :param measurement_station: A simplified object to represent the IEA Wind Task 43 WRA Data Model. + :type measurement_station: bw.MeasurementStation + :param wdir_cols: Wind direction column names to apply the offset to. If empty, all wind + direction columns in the data are used. Default is an empty list. + :type wdir_cols: list + :param inplace: If True, modifies `data` in place. If False, returns a new + DataFrame/Series with adjusted values. Default is False. + :type inplace: bool, optional + :param return_results_table: If True, returns a DataFrame containing the device orientation, + logger orientation and offset applied for each relevant time period. + :type return_results_table: bool, optional + :return: Data with wind direction adjusted by the orientation offset. + :rtype: pd.DataFrame | pd.Series | Tuple[pd.DataFrame | pd.Series, pd.DataFrame] **Example usage** :: @@ -1549,6 +1554,14 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp :: bw.apply_device_orientation_offset(data, fl1, inplace=True) print('Wind direction device orientation offset adjustment is completed.') + + Return the orientation offset results table along with adjusted timeseries data:: + :: + # Adjust only specific wind direction columns: + data_dev_orient_adj, results_table = bw.apply_device_orientation_offset( + data, fl1, wdir_cols=['Dir_40m', 'Dir_50m'], return_results_table=True + ) + results_table """ @@ -1580,6 +1593,7 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp _check_vertical_profiler_properties_overlap(measurement_station, df) + rows = [] # Apply the offset for i, wdir_prop in enumerate(wdirs_properties): name = wdir_prop['name'] @@ -1649,6 +1663,17 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp df[name] = _apply_dir_offset_target_orientation( df[name], logger_offset, device_orientation_deg, apply_offset_from, apply_offset_to, target_orientation_name='device orientation') + + height = wdir_prop.get('height_m') + rows.append({ + "Name": name, + "Height [m]": height, + "Device Orientation [deg]": device_orientation_deg, + "Logger Offset": logger_offset, + "Offset Applied [deg]": offset_wind_direction(device_orientation_deg, - logger_offset), + "Date From": apply_offset_from, + "Date To": apply_offset_to + }) else: wdir_not_in_dataset = True col_not_in_data.append(name) @@ -1669,6 +1694,34 @@ def apply_device_orientation_offset(data, measurement_station, wdir_cols=[], inp if isinstance(data, pd.Series): df = df[df.columns[0]] data.update(df) + + if return_results_table: + results_df = pd.DataFrame(rows).sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + results_df['consecutive_group'] = ( + (results_df['Name'] != results_df['Name'].shift()) | + (results_df['Height [m]'] != results_df['Height [m]'].shift()) | + (results_df['Device Orientation [deg]'] != results_df['Device Orientation [deg]'].shift()) | + (results_df['Logger Offset'] != results_df['Logger Offset'].shift()) | + (results_df['Offset Applied [deg]'] != results_df['Offset Applied [deg]'].shift()) + ).cumsum() + + # Group and aggregate consecutive periods with the same device orientation, logger offset and applied offset. + results_table = results_df.groupby([ + 'Name', + 'consecutive_group', + 'Height [m]', + 'Device Orientation [deg]', + 'Logger Offset', + 'Offset Applied [deg]' + ]).agg({ + 'Date From': 'first', + 'Date To': lambda x: None if any(d is None for d in x) else max(x) + }).reset_index(drop=False).drop(columns=['consecutive_group']).set_index("Name").sort_values( + by=["Height [m]", "Date From"], ascending=[False, True] + ) + return df, results_table return df From d5826fffbc198c6ed132d067e1a99a1c766498cf Mon Sep 17 00:00:00 2001 From: dancasey-ie Date: Fri, 28 Nov 2025 19:00:50 +0300 Subject: [PATCH 20/27] =?UTF-8?q?iss=20#550=20include=20method=20of=20auth?= =?UTF-8?q?enticating=20Brighthub=20requests=20using=20AP=E2=80=A6=20(#553?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * iss #550 include method of authenticating Brighthub requests using API keys. Included depreciation warnings for basic auth. Handle basic auth errors better. * Update docstrings * Update docstrings * iss #550 add entry to changelog * iss #550 PEP8, spelling and formatting updates * Change depreciation warnings to FutureWarnings * iss #550 Revert changelog changes, keep refresh token as depreciation warning * iss #550 update version number in warning * iss #550 put back in two missing changelog items --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 18 +-- brightwind/load/load.py | 263 +++++++++++++++++++++++++++------------- 2 files changed, 192 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e3b0536..e02535b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,17 @@ Additional labels for pre-release and build metadata are available as extensions 1. ### New Features and Enhancements -1. Added `scale_air_pressure_to_height` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) -2. Added `scale_air_density_to_height` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) -3. Added `scale_air_temperature_to_height` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) -4. Updated `calc_air_density` to include relative humidity as suggested in IEC 61400-12-1 ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) -5. Added `apply_scale_factor` to scale data by the scale_factor ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) -1. Added optional output to `apply_wind_vane_deadband_offset()` which provides a results table showing the applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) -1. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). +1. Added `scale_air_pressure_to_height()` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height. ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) +2. Added `scale_air_density_to_height()` to output an air density value for any height by applying a constant lapse rate to a known reference air density value at a reference measurement height. ([#534](https://github.com/brightwind-dev/brightwind/issues/534)) +3. Added `scale_air_temperature_to_height()` to output an air temperature value for any height by applying a constant lapse rate to a known reference air temperature value at a reference measurement height. ([#530](https://github.com/brightwind-dev/brightwind/issues/530)) +4. Updated `calc_air_density()` to include relative humidity as suggested in IEC 61400-12-1. ([#535](https://github.com/brightwind-dev/brightwind/issues/535)) + 1. Added deprecation warning the `specific_gas_constant` argument of `calc_air_density()` will be removed in v3.0. + 1. Added deprecation warning the scaling of air density to height within `calc_air_density()` will be removed in v3.0. Users should use `scale_air_density_to_height()` separately instead. +5. Added `apply_scale_factor()` to scale data by the scale_factor. ([#541](https://github.com/brightwind-dev/brightwind/issues/541)) +6. Added optional output to `apply_wind_vane_deadband_offset()` which provides a results table showing the applied offset. ([#520](https://github.com/brightwind-dev/brightwind/issues/520)) +7. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). +8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) + 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. ## [2.3.0] diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 01f85d76..d3cffd3f 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -903,7 +903,6 @@ class _BrighthubAuth: This class is used to define general functions that are then called by LoadBrightHub. Functions in this class are outside of LoadBrightHub and will be called only once during the analysis and this will avoid making multiple login to the Brighthub user pool. - """ # List possible errors encountered on Login @@ -914,12 +913,61 @@ class are outside of LoadBrightHub and will be called only once during the analy "new_password_required": "Your password has expired or needs to be reset. " "Kindly reset your Brighthub password and try again.", "password_not_verified": "Could not verify your password. " - "Please ensure you have confirmed your email and the password is correct." + "Please ensure you have confirmed your email and the password is correct.", + "ms_sso_user": "Microsoft SSO users cannot authenticate using 'BRIGHTHUB_EMAIL' and 'BRIGHTHUB_PASSWORD'. " + "Please migrate to API key authentication." } ID_TOKEN = '' REFRESH_TOKEN = '' USERNAME = '' PASSWORD = '' + CLIENT_ID = '' + CLIENT_SECRET = '' + + class BrighthubAuthError(Exception): + """Custom exception for Brighthub authentication failures.""" + pass + + @staticmethod + def _authenticate_with_client_credentials(): + """ + Authenticate a Brighthub user using the OAuth2 client credentials flow. + + This method retrieves the Brighthub client ID and client secret from environmental + variables if not already assigned in the _BrighthubAuth class. + + :rtype: str + :return: The JWT ID token for authenticated requests. + """ + + if not _BrighthubAuth.CLIENT_ID: + _BrighthubAuth.CLIENT_ID = utils.get_environment_variable('BRIGHTHUB_CLIENT_ID') + + if not _BrighthubAuth.CLIENT_SECRET: + _BrighthubAuth.CLIENT_SECRET = utils.get_environment_variable('BRIGHTHUB_CLIENT_SECRET') + + base_uri = os.getenv('BRIGHTHUB_BASE_URI', 'https://api.brighthub.io') + auth_url = f"{base_uri}/auth/token" + try: + response = requests.post( + url=auth_url, + auth=requests.auth.HTTPBasicAuth(_BrighthubAuth.CLIENT_ID, _BrighthubAuth.CLIENT_SECRET), + data={"grant_type": "client_credentials"}, + timeout=10 + ) + response.raise_for_status() + except requests.exceptions.RequestException as e: + raise _BrighthubAuth.BrighthubAuthError(f"Network or connection error during authentication: {e}") + + try: + token_data = response.json() + except ValueError: + raise _BrighthubAuth.BrighthubAuthError(f"Invalid JSON response from Brighthub: {response.text}") + + if 'id_token' not in token_data: + raise _BrighthubAuth.BrighthubAuthError(f"Authentication failed, no id_token returned: {token_data}") + + return token_data['id_token'] @staticmethod def _get_cognito_request(): @@ -932,27 +980,33 @@ def _get_cognito_request(): 'Content-Type': 'application/x-amz-json-1.1' } client_id = os.getenv("BRIGHTHUB_USER_POOL_CLIENT_ID", "3qkkpikve578cbok46p136au3g") - # client_id = utils.get_environment_variable('BRIGHTHUB_USER_POOL_CLIENT_ID') return url, headers, client_id - + @staticmethod - def _get_id_token(): - """ - Function to login to the Brighthub user pool. - Assign a id_token and a refresh_token to the global variables ID_TOKEN, REFRESH_TOKEN which can be used to - make requests to the APIs. - In case of an error, a error message will be returned + def _authenticate_with_basic_auth(): + """Authenticate a Brighthub user using username and password (USER_PASSWORD_AUTH flow) + via Cognito and return the ID token and refresh token. + :rtype: tuple + :return: A tuple containing the JWT ID token and refresh token for authenticated requests. """ - url, headers, client_id = _BrighthubAuth._get_cognito_request() - + warnings.warn( + "Authentication using 'BRIGHTHUB_EMAIL' and 'BRIGHTHUB_PASSWORD' is deprecated in v2.4.0 and " + "will be removed in v3.0.0. \nPlease migrate to API key authentication. " + "Create and manage API keys at: https://brighthub.io/account-settings/settings. " + "After generating a key, set the environment variables 'BRIGHTHUB_CLIENT_ID' and " + "'BRIGHTHUB_CLIENT_SECRET'.", + FutureWarning + ) if not _BrighthubAuth.USERNAME: _BrighthubAuth.USERNAME = utils.get_environment_variable('BRIGHTHUB_EMAIL') if not _BrighthubAuth.PASSWORD: _BrighthubAuth.PASSWORD = utils.get_environment_variable('BRIGHTHUB_PASSWORD') + url, headers, client_id = _BrighthubAuth._get_cognito_request() + body = { "AuthParameters": { "USERNAME": _BrighthubAuth.USERNAME, @@ -961,34 +1015,86 @@ def _get_id_token(): "AuthFlow": "USER_PASSWORD_AUTH", "ClientId": client_id } - response = requests.post(url, headers=headers, json=body) login_response = response.json() + login_response_type = login_response.get("__type") # a login error occurred - if login_response.get("__type"): - if login_response["__type"] == "NotAuthorizedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) - elif login_response["__type"] == "UserNotConfirmedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) + if login_response_type: + if login_response_type == "NotAuthorizedException": + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) + elif login_response_type == "UserNotConfirmedException": + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) + elif 'This account is linked to Microsoft SSO' in login_response.get("message", ""): + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["ms_sso_user"]) else: - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) # challenge returned if login_response.get("ChallengeName"): if login_response["ChallengeName"] == "NEW_PASSWORD_REQUIRED": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["new_password_required"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["new_password_required"]) elif login_response["ChallengeName"] == "PASSWORD_VERIFIER": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["password_verifier"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["password_verifier"]) else: - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) + raise _BrighthubAuth.BrighthubAuthError( + _BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) # login successful id_token = login_response['AuthenticationResult']['IdToken'] refresh_token = login_response['AuthenticationResult']['RefreshToken'] + return id_token, refresh_token - _BrighthubAuth.ID_TOKEN = id_token - _BrighthubAuth.REFRESH_TOKEN = refresh_token + @staticmethod + def _get_id_token(): + """ + Retrieve an ID token for authenticating BrightHub API requests. + + This method attempts to authenticate using API client credentials if + `BRIGHTHUB_CLIENT_ID` and `BRIGHTHUB_CLIENT_SECRET` are available. If they + are not provided, it falls back to username/password authentication using + `BRIGHTHUB_EMAIL` and `BRIGHTHUB_PASSWORD`. + + Username/password authentication is deprecated in v2.4.0 and will be removed in + v3.0.0. Users should migrate to API key–based authentication. + + Upon success, this method stores the resulting ID token (and refresh token, + when applicable) in the class-level attributes `ID_TOKEN` and + `REFRESH_TOKEN`. In case of authentication failure, an appropriate + exception is raised by the underlying authentication method. + + :rtype: dict + :return: An empty dictionary upon successful authentication. + """ + if _BrighthubAuth.REFRESH_TOKEN: + try: + _BrighthubAuth._brighthub_refresh_token() + return {} + except Exception: + # If refresh fails, proceed to re-authenticate + pass + + # Preferred authentication flow: client credentials (API key) + try: + _BrighthubAuth.ID_TOKEN = _BrighthubAuth._authenticate_with_client_credentials() + return {} + + except Exception as e: + if str(e) in [ + f'BRIGHTHUB_CLIENT_ID environmental variable is not set.', + f'BRIGHTHUB_CLIENT_SECRET environmental variable is not set.' + ]: + # Fallback to basic auth if client credentials are not provided + # This method is depreciated in v2.4.0 and will be removed in v3.0.0 + _BrighthubAuth.ID_TOKEN, _BrighthubAuth.REFRESH_TOKEN = _BrighthubAuth._authenticate_with_basic_auth() + else: + raise e return {} @@ -997,17 +1103,22 @@ def _brighthub_refresh_token(): """ Function to generate a new token if the current id_token has expired. The new tokens are assigned to the global variables ID_TOKEN, REFRESH_TOKEN. - In case of an error, a error message will be returned - + In case of an error, an error message will be returned. """ - url, headers, client_id = _BrighthubAuth._get_cognito_request() + warnings.warn( + "Refresh token authentication is depreciated in v2.4.0 and will be removed in v3.0.0. " + "Please migrate to API key authentication.", + DeprecationWarning, + stacklevel=3 + ) + url, headers, cognito_client_id = _BrighthubAuth._get_cognito_request() body = { "AuthParameters": { "REFRESH_TOKEN": _BrighthubAuth.REFRESH_TOKEN }, "AuthFlow": "REFRESH_TOKEN_AUTH", - "ClientId": client_id + "ClientId": cognito_client_id } response = requests.post(url, headers=headers, json=body) login_response = response.json() @@ -1015,17 +1126,16 @@ def _brighthub_refresh_token(): # a login error occurred if login_response.get("__type"): if login_response["__type"] == "NotAuthorizedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) + raise ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["not_authorized"]) elif login_response["__type"] == "UserNotConfirmedException": - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) + raise ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["user_not_confirmed"]) else: - return ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) + raise ImportError(_BrighthubAuth.__BRIGHTHUB_LOGIN_ERROR_MAP["unexpected_error"]) id_token = login_response['AuthenticationResult']['IdToken'] _BrighthubAuth.ID_TOKEN = id_token new_refresh_token = "" - # refresh token only expires after 30 days, it is not returned in the response if it is still valid if login_response['AuthenticationResult'].get('RefreshToken'): new_refresh_token = login_response['AuthenticationResult']['RefreshToken'] @@ -1039,20 +1149,20 @@ def _brighthub_refresh_token(): class LoadBrightHub: """ - LoadBrightHub allows you to pull meta data and timeseries data of measurements from the BrightHub + LoadBrightHub allows you to pull metadata and timeseries data of measurements from the BrightHub platform. This is a fast way to get access to the available open datasets on the platform. - To use LoadBrightHub, first sign up on www.brighthub.io and note your email and password. - - For security purposes LoadBrightHub uses stored environmental variables for your log in details. The - BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD environmental variables need to be set. In Windows this can be - done by opening the command prompt in Administrator mode and running: - - > setx BRIGHTHUB_EMAIL "your email") - > setx BRIGHTHUB_PASSWORD "your password") + To use LoadBrightHub: + 1. Create a BrightHub account at https://brighthub.io/auth/create-account. + 2. Create a new API key at https://brighthub.io/account-settings/settings. + and note your Client ID and Client Secret. + 3. Set the BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET as environmental variables. + In Windows this can be done by opening the command prompt in Administrator mode and running: + > setx BRIGHTHUB_CLIENT_ID "your API key client id") + > setx BRIGHTHUB_CLIENT_SECRET "your API key client secret") - If Anaconda or your Python environment is running you will need to restart it for the environmental variables to - take effect. + If Anaconda or your Python environment is running you will need to restart it for + the environmental variables to take effect. You can start by pulling all the available measurement stations available to you by running: @@ -1061,7 +1171,6 @@ class LoadBrightHub: """ __BASE_URI = os.getenv('BRIGHTHUB_BASE_URI', 'https://api.brighthub.io') - # __BASE_URI = utils.get_environment_variable('BRIGHTHUB_BASE_URI') @staticmethod def _brighthub_request(url_end, params=None): @@ -1075,11 +1184,9 @@ def _brighthub_request(url_end, params=None): :return response: The requests response object returned by requests.get() :rtype: requests.Response object """ - + if not _BrighthubAuth.ID_TOKEN: - login_response = _BrighthubAuth._get_id_token() - if "error" in login_response: - return ImportError(login_response) + _BrighthubAuth._get_id_token() url = "{}{}".format(LoadBrightHub.__BASE_URI, url_end) headers = {"authorization": _BrighthubAuth.ID_TOKEN} @@ -1088,19 +1195,11 @@ def _brighthub_request(url_end, params=None): # If there is an auth error due to expired token if response.status_code == 401 and response.json().get("message") == "The incoming token has expired": - # generate the token again - refresh_token_response = _BrighthubAuth._brighthub_refresh_token() - - # if an error occurred - if refresh_token_response.get("error"): - return ImportError(refresh_token_response) - else: - # token refreshed successfully - headers = {"authorization": _BrighthubAuth.ID_TOKEN} - - # make the request again - response = requests.get(url=url, headers=headers, params=params) - return response + # Authenticate again + _BrighthubAuth._get_id_token() + headers = {"authorization": _BrighthubAuth.ID_TOKEN} + # Make the request again + response = requests.get(url=url, headers=headers, params=params) return response @@ -1116,17 +1215,17 @@ def get_plants(plant_type=None, plant_uuid=None): :return: A table showing the available plants. :rtype: pd.DataFrame - To use LoadBrightHub, first sign up on www.brighthub.io and note your email and password. - - For security purposes LoadBrightHub uses stored environmental variables for your log in details. The - BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD environmental variables need to be set. In Windows this can be - done by opening the command prompt in Administrator mode and running: - - > setx BRIGHTHUB_EMAIL "your email") - > setx BRIGHTHUB_PASSWORD "your password") + To use LoadBrightHub: + 1. Create a BrightHub account at https://brighthub.io/auth/create-account. + 2. Create a new API key at https://brighthub.io/account-settings/settings. + and note your Client ID and Client Secret. + 3. Set the BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET as environmental variables. + In Windows this can be done by opening the command prompt in Administrator mode and running: + > setx BRIGHTHUB_CLIENT_ID "your API key client id") + > setx BRIGHTHUB_CLIENT_SECRET "your API key client secret") - If Anaconda or your Python environment is running you will need to restart it for the environmental variables to - take effect. + If Anaconda or your Python environment is running you will need to restart it for + the environmental variables to take effect. **Example usage** :: @@ -1192,17 +1291,17 @@ def get_measurement_stations(plant_uuid=None, measurement_station_uuid=None, mea :return: A table showing the available measurement stations. :rtype: pd.DataFrame | List[dict] - To use LoadBrightHub, first sign up on www.brighthub.io and note your email and password. - - For security purposes LoadBrightHub uses stored environmental variables for your log in details. The - BRIGHTHUB_EMAIL and BRIGHTHUB_PASSWORD environmental variables need to be set. In Windows this can be - done by opening the command prompt in Administrator mode and running: - - > setx BRIGHTHUB_EMAIL "your email") - > setx BRIGHTHUB_PASSWORD "your password") - - If Anaconda or your Python environment is running you will need to restart it for the environmental variables to - take effect. + To use LoadBrightHub: + 1. Create a BrightHub account at https://brighthub.io/auth/create-account. + 2. Create a new API key at https://brighthub.io/account-settings/settings. + and note your Client ID and Client Secret. + 3. Set the BRIGHTHUB_CLIENT_ID and BRIGHTHUB_CLIENT_SECRET as environmental variables. + In Windows this can be done by opening the command prompt in Administrator mode and running: + > setx BRIGHTHUB_CLIENT_ID "your API key client id") + > setx BRIGHTHUB_CLIENT_SECRET "your API key client secret") + + If Anaconda or your Python environment is running you will need to restart it for + the environmental variables to take effect. **Example usage** :: From ba270302a1bfdad7eb9d1058e56546c4fdc0bfb4 Mon Sep 17 00:00:00 2001 From: olivia-bentley Date: Mon, 1 Dec 2025 16:45:19 +0000 Subject: [PATCH 21/27] iss #545 added argument minimum_gap_length to time_continuity_gaps (#546) * iss #545 added argument minimum_gap_length to time_continuity_gaps * iss #545 update changelog * iss #545 added more filtering examples * iss #545 update spacing * iss #545 update doc string to clarify it is a 'time' gap * iss #545 update changelog --------- Co-authored-by: stephenholleran --- CHANGELOG.md | 3 ++- brightwind/analyse/analyse.py | 34 +++++++++++++++++++++++++++------- tests/test_analyse.py | 6 ++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e02535b7..aec12fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0-dev] -1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)). ### Bug Fixes @@ -30,6 +30,7 @@ Additional labels for pre-release and build metadata are available as extensions 7. Added optional output to `apply_device_orientation_offset()` which provides a results table showing the applied offset. ([#521](https://github.com/brightwind-dev/brightwind/issues/521)). 8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. +1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) ## [2.3.0] diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 0c5d746c..3877f96b 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -1,5 +1,6 @@ import pandas as pd import numpy as np +from typing import Optional from brightwind.transform import transform as tf from brightwind.utils import utils from brightwind.analyse import plot as bw_plt @@ -1369,7 +1370,7 @@ def freq_table(var_series, direction_series, var_bin_array=np.arange(-0.5, 41, 1 return graph -def time_continuity_gaps(data): +def time_continuity_gaps(data: pd.DataFrame, minimum_gap_length: Optional[pd.Timedelta] = None): """ Returns a table listing all the time gaps in the data that are not equal to the derived temporal resolution. @@ -1394,20 +1395,34 @@ def time_continuity_gaps(data): in the two available timestamps. It gives the actual amount of data missing e.g. if the two timestamps were 2020-01-01 01:10 and 2020-01-01 01:50 the days lost will equate to a 30 min of missing data and not 40 min. - :param data: Data for checking continuity, timestamp must be the index - :type data: pd.Series or pd.DataFrame - :return: A table listing all the time gaps in the data that are not equal to the derived - temporal resolution. - :rtype: pd.DataFrame + :param data: Data for checking continuity, timestamp must be the index + :type data: pd.Series or pd.DataFrame + :param minimum_gap_length: The minimum length time gap to report. Shorter time gaps will be filtered + out of the returned DataFrame + :type minimum_gap_length: Optional[pd.Timedelta] + :return: A table listing all the time gaps in the data that are not equal to the + derived temporal resolution. + :rtype: pd.DataFrame **Example usage** :: + import pandas as pd import brightwind as bw + data = bw.load_csv(bw.demo_datasets.demo_data) bw.time_continuity_gaps(data) bw.time_continuity_gaps(data['Spd80mN']) + # Removing all gaps shorter than 4hrs 30mins + bw.time_continuity_gaps(data['Spd80mN'], pd.Timedelta("4h 30min")) + + # Removing all gaps shorter than 20mins, expressed using alternative pandas notation + bw.time_continuity_gaps(data['Spd80mN'], pd.Timedelta(minutes=20)) + + # Removing all gaps shorter than 1day + bw.time_continuity_gaps(data['Spd80mN'], pd.Timedelta(days=1)) + """ indexes = data.dropna(how='all').index resolution = tf._get_data_resolution(indexes) @@ -1417,7 +1432,12 @@ def time_continuity_gaps(data): continuity = pd.DataFrame({'Date From': indexes.values.flatten()[:-1], 'Date To': indexes.values.flatten()[1:]}) - continuity['Days Lost'] = (continuity['Date To'] - continuity['Date From']) / pd.Timedelta('1 days') + continuity['Time gap'] = (continuity['Date To'] - continuity['Date From']) + + if minimum_gap_length: + continuity = continuity[continuity['Time gap'] >= minimum_gap_length] + + continuity['Days Lost'] = continuity['Time gap'] / pd.Timedelta('1 days') # Remove indexes where no days are lost before returning diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 2a3cf01a..bc1f5159 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -152,6 +152,12 @@ def test_time_continuity_gaps(): assert gaps.iloc[1, 1] == pd.Timestamp('2016-03-30 07:10:00') assert abs(gaps.iloc[0, 2] - 0.173611) < 1e-5 assert abs(gaps.iloc[1, 2] - 0.305556) < 1e-5 + + gaps = bw.time_continuity_gaps(DATA['Spd80mN'], pd.Timedelta("4h 30min")) + assert gaps.iloc[0, 0] == pd.Timestamp('2016-03-29 23:40:00') + assert gaps.iloc[0, 1] == pd.Timestamp('2016-03-30 07:10:00') + assert gaps.iloc[1, 0] == pd.Timestamp('2016-05-11 23:00:00') + assert gaps.iloc[1, 1] == pd.Timestamp('2016-05-31 15:20:00') # test for when timesteps are irregular # THIS WILL RAISE 3 WARNINGS. From 609374eddee05646eb5f96d08103fb13dff7ba3b Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:02:46 +0000 Subject: [PATCH 22/27] [Iss504] fix bug offset timestamps (#518) * iss #504 updated offset_timestamps function to include last timestamp when date_from unspecified * Revert "iss #504 updated offset_timestamps function to include last timestamp when date_from unspecified" This reverts commit f8040f691540dfc6d57ed6a30c6cbf3b8f62e6c2. * iss #504 updated offset_timestamps function to include last timestamp when date_to unspecified by setting date_to as last timestamp in dataframe + 10 mins * iss #504 added tests for this scenario to test_transform.py * iss #504 updated to add 1 second to the last timestamp instead of 10mins when no date_to given --------- Co-authored-by: Sara Rafter Co-authored-by: stephenholleran --- CHANGELOG.md | 4 ++-- brightwind/transform/transform.py | 4 ++-- tests/test_transform.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec12fed..5289a915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions --- ## [2.4.0-dev] -1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)). - +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) ### Bug Fixes 1. @@ -31,6 +30,7 @@ Additional labels for pre-release and build metadata are available as extensions 8. Updated `LoadBrightHub()` to use BrightHub API Keys. ([#550](https://github.com/brightwind-dev/brightwind/issues/550)) 1. Added deprecation warning the username and password method of authenticating in BrightHub will be removed in a future version. 1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) +1. Updated `offset_timestamps()`to include last timestamp when date_to is unspecified, so that offset is applied to the entire record if date_to not specified. ([#504](https://github.com/brightwind-dev/brightwind/issues/504)) ## [2.3.0] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index b123aaa9..f9ce855d 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -1441,9 +1441,9 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals if pd.isnull(date_to): if isinstance(data, pd.DatetimeIndex): - date_to = data[-1] + date_to = data[-1] + pd.DateOffset(seconds=1) elif isinstance(data, pd.Series) or isinstance(data, pd.DataFrame): - date_to = data.index[-1] + date_to = data.index[-1] + pd.DateOffset(seconds=1) else: date_to = pd.to_datetime(date_to) diff --git a/tests/test_transform.py b/tests/test_transform.py index 3a1c6ebb..932442d5 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -357,6 +357,24 @@ def test_offset_timestamps(): # sending index with no start end bw.offset_timestamps(series1.index, offset='90min') + series2 = DATA['2016-01-10 00:00:00':'2017-01-10 00:00:00'] + op = bw.offset_timestamps(series2.index, offset='90min') + assert len(op) == len(series2) + assert op[0] == pd.to_datetime('2016-01-10 01:30:00') + assert op[-1] == pd.to_datetime('2017-01-10 01:30:00') + + op = bw.offset_timestamps(series2.index, offset='-2H') + assert len(op) == len(series2) + assert op[0] == pd.to_datetime('2016-01-09 22:00:00') + assert op[-1] == pd.to_datetime('2017-01-09 22:00:00') + + # sending DataFrame with datetime index with no start end + op = bw.offset_timestamps(series2, offset='-10min') + assert (op.iloc[0] == series2.iloc[0]).all() + assert (op.iloc[-1] == series2.iloc[-1]).all() + assert len(op) == len(series2) + assert (op.loc['2017-01-09 23:50:00'] == series2.loc['2017-01-10 00:00:00']).all() + # sending index with start end op = bw.offset_timestamps(series1.index, offset='2min', date_from='2016-01-10 00:10:00') assert op[0] == pd.to_datetime('2016-01-10 00:00:00') From 586fa6f14709460d92ea59001913f97b302d7057 Mon Sep 17 00:00:00 2001 From: rm-mol <93541751+rm-mol@users.noreply.github.com> Date: Mon, 15 Dec 2025 19:50:08 +0100 Subject: [PATCH 23/27] iss255 tackle deprecation warnings (#517) * Update python, numpy, pandas * WIP * WIP * WIP * WIP * WIP * github actions only for Python 3.12 * add env variable * add tests.yml from dev branch * fix PEP8 missing whitespace after comma * missed PEP 8 whitespace * add backward compatibility and deprecating warnings for A, AS and H * add deprecating warning for T * add deprecating warning for S and backward compatibility * remove T from acceptable list * add msg of remaining actions * iss #255 compatibility testing and inclusion of dataframe_map * iss #255 add deprecation warning to `dataframe_map` function * iss #255 revised stacklevel of warning * iss #255 handling previously missed deprecation on time strings * iss #255 tidied comments * iss #255 updated docstring * iss #255 remove deprecated test * only convert string if necessary * changelog edit required * update changelog --------- Co-authored-by: stephenholleran Co-authored-by: r-molins-mrp <93541751+r-molins-mrp@users.noreply.github.com> Co-authored-by: Olivia Bentley --- CHANGELOG.md | 2 + brightwind/analyse/analyse.py | 29 ++- brightwind/analyse/correlation.py | 39 +-- brightwind/analyse/plot.py | 4 +- brightwind/analyse/shear.py | 10 +- brightwind/load/load.py | 6 +- brightwind/load/station.py | 2 +- brightwind/transform/transform.py | 227 ++++++++++++++++-- .../how_to_get_some_useful_stats.ipynb | 4 +- .../how_to_transform_your_data.ipynb | 8 +- setup.py | 3 +- tests/test_analyse.py | 10 +- tests/test_correlation.py | 27 ++- tests/test_shear.py | 8 +- tests/test_transform.py | 128 +++++++--- 15 files changed, 378 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5289a915..30ed7d16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Additional labels for pre-release and build metadata are available as extensions ## [2.4.0-dev] 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) + ### Bug Fixes 1. diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index 3877f96b..bce9e967 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -154,10 +154,11 @@ def custom_agg(x): axis=1).dropna() if aggregation_method == '%frequency': - counts = data.groupby([y_series.name, x_series.name]).count().unstack(level=-1) + counts = data.groupby([y_series.name, x_series.name], observed=False).count().unstack(level=-1) distribution = counts / (counts.sum().sum()) * 100.0 else: - distribution = data.groupby([y_series.name, x_series.name]).agg(aggregation_method).unstack(level=-1) + distribution = data.groupby([y_series.name, x_series.name], + observed=False).agg(aggregation_method).unstack(level=-1) if y_bin_labels is not None: distribution.index = y_bin_labels @@ -390,7 +391,7 @@ def _mean_of_monthly_means_basic_method(df: pd.Series) -> pd.Series: Return a Series of mean of monthly mean with timestamp as the index. Calculate the monthly mean for each calendar month and then average the resulting 12 months. """ - mean_monthly_mean: pd.Series = df.groupby(df.index.month).mean().mean() + mean_monthly_mean: pd.Series = df.groupby(df.index.month, observed=False).mean().mean() return mean_monthly_mean @@ -618,9 +619,10 @@ def _derive_distribution(var_to_bin, var_to_bin_against, bins=None, aggregation_ data = pd.concat([var_to_bin.rename('data'), var_binned_series], join='inner', axis=1) if aggregation_method == '%frequency': - distribution = data.groupby(['variable_bin'])['data'].count().rename('%frequency') / len(data) * 100.0 + distribution = data.groupby(['variable_bin'], + observed=False)['data'].count().rename('%frequency') / len(data) * 100.0 else: - distribution = data.groupby(['variable_bin'])['data'].agg(aggregation_method) + distribution = data.groupby(['variable_bin'], observed=False)['data'].agg(aggregation_method) return distribution @@ -883,9 +885,9 @@ def dist_by_dir_sector(var_series, direction_series, sectors=12, aggregation_met _get_direction_binned_series(sectors, direction_series, direction_bin_array, direction_bin_labels) data = pd.concat([var_series.rename('data'), direction_binned_series], join='inner', axis=1) if aggregation_method == '%frequency': - result = data.groupby(['direction_bin'])['data'].count().rename('%frequency')/len(data) * 100.0 + result = data.groupby(['direction_bin'], observed=False)['data'].count().rename('%frequency')/len(data) * 100.0 else: - result = data.groupby(['direction_bin'])['data'].agg(aggregation_method) + result = data.groupby(['direction_bin'], observed=False)['data'].agg(aggregation_method) for i in range(1, sectors+1): if not (i in result.index): @@ -923,10 +925,11 @@ def _get_dist_matrix_by_dir_sector(var_series, var_to_bin_series, direction_seri data = pd.concat([var_series.rename('var_data'), var_binned_series, direction_binned_series], axis=1).dropna() if aggregation_method == '%frequency': - counts = data.groupby([var_to_bin_series.name, 'direction_bin']).count().unstack(level=-1) + counts = data.groupby([var_to_bin_series.name, 'direction_bin'], observed=False).count().unstack(level=-1) distribution = counts/(counts.sum().sum()) * 100.0 else: - distribution = data.groupby([var_to_bin_series.name, 'direction_bin']).agg(aggregation_method).unstack(level=-1) + distribution = data.groupby([var_to_bin_series.name, 'direction_bin'], + observed=False).agg(aggregation_method).unstack(level=-1) distribution.columns = distribution.columns.droplevel(0) for i in range(1, sectors + 1): if not (i in distribution.columns): @@ -1482,11 +1485,11 @@ def coverage(data, period='1M', aggregation_method='mean', data_resolution=None) :param period: Groups data by the period specified here. The following formats are supported - Set period to 10min for 10 minute average, 20min for 20 minute average and so on for 4min, 15min, etc. - - Set period to 1H for hourly average, 3H for three hourly average and so on for 5H, 6H etc. + - Set period to 1h for hourly average, 3h for three hourly average and so on for 5h, 6h etc. - Set period to 1D for a daily average, 3D for three day average, similarly 5D, 7D, 15D etc. - Set period to 1W for a weekly average, 3W for three week average, similarly 2W, 4W etc. - Set period to 1M for monthly average - - Set period to 1AS fo annual average + - Set period to 1YS fo annual average - Can be a DateOffset object too :type period: str or pandas.DateOffset @@ -1506,10 +1509,10 @@ def coverage(data, period='1M', aggregation_method='mean', data_resolution=None) data = bw.load_campbell_scientific(bw.demo_datasets.demo_campbell_scientific_site_data) #To find hourly coverage - data_hourly = bw.coverage(data.Spd80mN, period='1H') + data_hourly = bw.coverage(data.Spd80mN, period='1h') #To find hourly coverage for multiple columns - data_hourly_multiple = bw.coverage(data[['Spd80mS','Spd60mN']], period='1H') + data_hourly_multiple = bw.coverage(data[['Spd80mS','Spd60mN']], period='1h') #To find monthly_coverage data_monthly = bw.coverage(data.Spd80mN, period='1M') diff --git a/brightwind/analyse/correlation.py b/brightwind/analyse/correlation.py index 9fe3e17e..63a0181e 100644 --- a/brightwind/analyse/correlation.py +++ b/brightwind/analyse/correlation.py @@ -220,9 +220,10 @@ def _rename_duplicated_columns(self): return self.ref_spd, self._ref_spd_col_name, self._ref_spd_col_names, self.ref_dir, self._ref_dir_col_name def _get_synth_start_dates(self): - none_even_freq = ['5H', '7H', '9H', '10H', '11H', '13H', '14H', '15H', '16H', '17H', '18H', '19H', - '20H', '21H', '22H', '23H', 'D', 'W'] + none_even_freq = ['5h', '7h', '9h', '10h', '11h', '13h', '14h', '15h', '16h', '17h', '18h', '19h', + '20h', '21h', '22h', '23h', 'D', 'W'] if any(freq in self.averaging_prd for freq in none_even_freq): + self.averaging_prd = tf._normalize_freq_string(self.averaging_prd) ref_time_array = pd.date_range(start=self.data.index[0], freq='-' + self.averaging_prd, end=self.ref_spd.index[0]) if ref_time_array.empty: @@ -323,11 +324,11 @@ class OrdinaryLeastSquares(CorrelBase): :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -405,7 +406,7 @@ class OrdinaryLeastSquares(CorrelBase): # Correlate temperature on an hourly basis using a different aggregation method. ols_cor = bw.Correl.OrdinaryLeastSquares(m2_ne['T2M_degC'], data['T2m'], - averaging_prd='1H', coverage_threshold=0, + averaging_prd='1h', coverage_threshold=0, ref_aggregation_method='min', target_aggregation_method='min') # Correlate wind speeds on a monthly basis and force the intercept through the origin. @@ -419,7 +420,7 @@ class OrdinaryLeastSquares(CorrelBase): # Correlate by directional sector forcing the intercept through the origin. ols_cor = bw.Correl.OrdinaryLeastSquares(m2_ne['WS50m_m/s'], data['Spd80mN'], - ref_dir=m2_ne['WD50m_deg'], averaging_prd='1H', + ref_dir=m2_ne['WD50m_deg'], averaging_prd='1h', coverage_threshold=0.9, forced_intercept_origin=True) """ def __init__(self, ref_spd, target_spd, averaging_prd, coverage_threshold=0.9, ref_dir=None, sectors=12, @@ -462,7 +463,7 @@ def run(self, show_params=True): elif type(self.ref_dir) is pd.Series: self.params = [] for sector, group in pd.concat([self.data, self._ref_dir_bins], - axis=1, join='inner').dropna().groupby('ref_dir_bin'): + axis=1, join='inner').dropna().groupby('ref_dir_bin', observed=False): # print('Processing sector:', sector) if len(group) > 1: slope, offset = self._leastsquare(ref_spd=group[self._ref_spd_col_name], @@ -514,11 +515,11 @@ class OrthogonalLeastSquares(CorrelBase): :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -583,7 +584,7 @@ class OrthogonalLeastSquares(CorrelBase): # Correlate temperature on an hourly basis using a different aggregation method. orthog_cor = bw.Correl.OrthogonalLeastSquares(m2_ne['T2M_degC'], data['T2m'], - averaging_prd='1H', coverage_threshold=0, + averaging_prd='1h', coverage_threshold=0, ref_aggregation_method='min', target_aggregation_method='min') """ @@ -638,11 +639,11 @@ class MultipleLinearRegression(CorrelBase): :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -705,7 +706,7 @@ class MultipleLinearRegression(CorrelBase): # Correlate temperature on an hourly basis using a different aggregation method. mul_cor = bw.Correl.MultipleLinearRegression([m2_ne['T2M_degC'], m2_nw['T2M_degC']], data['T2m'], - averaging_prd='1H', coverage_threshold=0, + averaging_prd='1h', coverage_threshold=0, ref_aggregation_method='min', target_aggregation_method='min') """ @@ -905,11 +906,11 @@ def __init__(self, ref_spd, ref_dir, target_spd, target_dir, averaging_prd, cove :param averaging_prd: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type averaging_prd: str :param coverage_threshold: Minimum coverage required when aggregating the data to the averaging_prd. @@ -936,7 +937,7 @@ def __init__(self, ref_spd, ref_dir, target_spd, target_dir, averaging_prd, cove # Basic usage on an hourly basis ss_cor = bw.Correl.SpeedSort(m2['WS50m_m/s'], m2['WD50m_deg'], data['Spd80mN'], data['Dir78mS'], - averaging_prd='1H') + averaging_prd='1h') ss_cor.run() ss_cor.plot_wind_directions() ss_cor.get_result_table() @@ -944,7 +945,7 @@ def __init__(self, ref_spd, ref_dir, target_spd, target_dir, averaging_prd, cove # Sending an array of direction sectors ss_cor = bw.Correl.SpeedSort(m2['WS50m_m/s'], m2['WD50m_deg'], data['Spd80mN'], data['Dir78mS'], - averaging_prd='1H', direction_bin_array=[0,90,130,200,360]) + averaging_prd='1h', direction_bin_array=[0,90,130,200,360]) ss_cor.run() """ @@ -1018,7 +1019,7 @@ def run(self, show_params=True): self.params['target_veer_cutoff'] = round(self.target_veer_cutoff, 5) self.params['overall_average_veer'] = round(self.overall_veer, 5) for sector, group in pd.concat([self.data, self._ref_dir_bins], - axis=1, join='inner').dropna().groupby('ref_dir_bin'): + axis=1, join='inner').dropna().groupby('ref_dir_bin', observed=False): # print('Processing sector:', sector) self.speed_model[sector] = SpeedSort.SectorSpeedModel(ref_spd=group[self._ref_spd_col_name], target_spd=group[self._tar_spd_col_name], @@ -1210,7 +1211,7 @@ def _predict(self, x_spd, x_dir): direction_bin_array=self.direction_bin_array).rename('ref_dir_bin')], axis=1, join='inner').dropna() prediction = pd.Series(dtype='float64').rename('spd') - for sector, data in x.groupby('ref_dir_bin'): + for sector, data in x.groupby('ref_dir_bin', observed=False): if sector in list(self.speed_model.keys()): prediction_spd = self.speed_model[sector].sector_predict(data['spd']) else: diff --git a/brightwind/analyse/plot.py b/brightwind/analyse/plot.py index 95303b59..2063f9f9 100644 --- a/brightwind/analyse/plot.py +++ b/brightwind/analyse/plot.py @@ -1971,7 +1971,7 @@ def plot_shear_by_sector(scale_variable, wind_rose_data, calc_method='power_law' if calc_method == 'log_law': label = 'Mean_Roughness_Coefficient' - scale_variable_y = np.append(scale_variable, scale_variable[0]) + scale_variable_y = np.append(scale_variable, scale_variable.iloc[0]) plot_x = np.append(radians, radians[0]) scale_to_fit = max(scale_variable[np.isfinite(scale_variable)]) / max(result / 100) wind_rose_r = (result / 100) * scale_to_fit @@ -2352,7 +2352,7 @@ def plot_shear_time_of_day(df, calc_method, plot_type='step'): ax.set_ylabel(label) # create x values for plot - idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1H').hour + idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1h').hour if plot_type == 'step': df = df.shift(+1, axis=0) diff --git a/brightwind/analyse/shear.py b/brightwind/analyse/shear.py index 592147f9..f49f5101 100644 --- a/brightwind/analyse/shear.py +++ b/brightwind/analyse/shear.py @@ -1044,13 +1044,13 @@ def _apply(self, wspds, height, shear_to, wdir=None): if self.calc_method == 'power_law': scaled_wspds[i] = Shear._scale(wspds=by_sector[i]['Unscaled_Wind_Speeds'], height=height, shear_to=shear_to, - calc_method=self.calc_method, alpha=self.alpha[i]) + calc_method=self.calc_method, alpha=self.alpha.iloc[i]) elif self.calc_method == 'log_law': scaled_wspds[i] = Shear._scale(wspds=by_sector[i]['Unscaled_Wind_Speeds'], height=height, shear_to=shear_to, calc_method=self.calc_method, - roughness=self._roughness[i]) + roughness=self._roughness.iloc[i]) if i == 0: result = scaled_wspds[i] @@ -1093,7 +1093,7 @@ def _fill_df_12x24(data): df_copy = data.copy() interval = int(24 / len(data)) # set index for new data frame to deal with less than 24 sectors - idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1H') + idx = pd.date_range('2017-01-01 00:00', '2017-01-01 23:00', freq='1h') # create new dataframe with 24 rows only interval number of unique values df = pd.DataFrame({cols: [np.nan] for cols in df_copy.columns}, index=pd.DatetimeIndex(idx).time) @@ -1130,13 +1130,13 @@ def _data_prep(wspds, heights, min_speed, maximise_data=False, return_raw_wspds= Shear._valid_wsp_data_error_msg(wspds, min_speed) if isinstance(wspds.index, pd.DatetimeIndex): if maximise_data is False: - cvg = coverage(wspds[wspds > min_speed].dropna(), period='1AS').sum()[1] + cvg = coverage(wspds[wspds > min_speed].dropna(), period='1YS').sum().iloc[1] else: _wspds = wspds[wspds > min_speed] count = _wspds.count(axis=1) count = count[count >= 2] count.rename('count', inplace=True) - cvg = coverage(count, period='1AS').sum() + cvg = coverage(count, period='1YS').sum() if not return_raw_wspds: wspds = wspds[wspds > min_speed].dropna() return wspds, cvg diff --git a/brightwind/load/load.py b/brightwind/load/load.py index d3cffd3f..7f02f700 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -1362,7 +1362,7 @@ def get_measurement_stations(plant_uuid=None, measurement_station_uuid=None, mea ) if return_df: - meas_loc_df = pd.read_json(json.dumps(meas_loc_json)) + meas_loc_df = pd.read_json(StringIO(json.dumps(meas_loc_json))) required_cols = ['name', 'measurement_station_type_id', 'latitude_ddeg', 'longitude_ddeg', 'plant_uuid', 'uuid', 'notes'] meas_loc_df = meas_loc_df[required_cols] @@ -2422,7 +2422,7 @@ def apply_cleaning(data, cleaning_file_or_df, inplace=False, sensor_col_name='Se else: for col in data.columns: if col.find(cleaning_df[sensor_col_name][k]) == 0: - data[col][(data.index >= date_from) & (data.index < date_to)] = replacement_text + data.loc[(data.index >= date_from) & (data.index < date_to), col] = replacement_text pd.options.mode.chained_assignment = 'warn' return data @@ -2651,7 +2651,7 @@ def apply_cleaning_windographer(data, windog_cleaning_file, inplace=False, flags for col in data.columns: if col.find(cleaning_df[sensor_col_name][k]) == 0: if cleaning_df[flag_col_name][k] not in flags_to_exclude: - data[col][(data.index >= date_from) & (data.index < date_to)] = replacement_text + data.loc[(data.index >= date_from) & (data.index < date_to), col] = replacement_text pd.options.mode.chained_assignment = 'warn' return data diff --git a/brightwind/load/station.py b/brightwind/load/station.py index 3a9bd29c..889ee7ab 100644 --- a/brightwind/load/station.py +++ b/brightwind/load/station.py @@ -903,7 +903,7 @@ def __get_table_for_cols(self, columns_to_show): temp_df.sort_values(['name', 'date_from'], ascending=[True, True], inplace=True) temp_df.fillna('-', inplace=True) # groupby drops nan so need to fill them in # group duplicate data for the columns available - grouped_by_avail_cols = temp_df.groupby(avail_cols) + grouped_by_avail_cols = temp_df.groupby(avail_cols, observed=False) # get date_to from the last row in each group to assign to the first row. new_date_to = grouped_by_avail_cols.last()['date_to'] df = grouped_by_avail_cols.first()[['date_from', 'date_to']] diff --git a/brightwind/transform/transform.py b/brightwind/transform/transform.py index f9ce855d..d8b53e49 100644 --- a/brightwind/transform/transform.py +++ b/brightwind/transform/transform.py @@ -24,6 +24,12 @@ 'apply_wspd_slope_offset_adj', 'apply_device_orientation_offset'] +_warned_a = False # warning for pandas 'A' frequency string so only shows once +_warned_as = False # warning for pandas 'AS' frequency string so only shows once +_warned_h = False # warning for pandas 'H' frequency string so only shows once +_warned_t = False # warning for pandas 'T' frequency string so only shows once +_warned_s = False # warning for pandas 'S' frequency string so only shows once + def _compute_wind_vector(wspd, wdir): """ @@ -32,6 +38,120 @@ def _compute_wind_vector(wspd, wdir): return wspd*np.cos(wdir), wspd*np.sin(wdir) +def dataframe_map(df, func, **kwargs): + """ + Apply a function element-wise to a DataFrame. + + Compatibility wrapper for DataFrame.map() (pandas >=2.1) and + DataFrame.applymap() (pandas <2.1). + + :param df: DataFrame to apply function to + :type df: pd.DataFrame + :param func: Function to apply element-wise + :type func: Callable + :return: Transformed DataFrame + :rtype: pd.DataFrame + """ + if hasattr(df, 'map'): + # pandas >= 2.1 + return df.map(func, **kwargs) + else: + # pandas < 2.1 + warnings.warn( + "Support for pandas versions <2.1 is ending in a future brightwind version.", + DeprecationWarning, + stacklevel=2 + ) + return df.applymap(func, **kwargs) + + +def _normalize_freq_string(period): + """ + Convert a deprecated pandas frequency string to its modern equivalent. + + Pandas frequency strings are available here: + https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects + + :param period: Frequency string that may or may not be deprecated + :type period: str + :return: Frequency string in it's modern equivalent form where necessary. + :rtype: str + """ + global _warned_a, _warned_as, _warned_h, _warned_t, _warned_s + pandas_version = tuple(map(int, pd.__version__.split('.')[:2])) + + # Handle deprecated 'A' -> 'Y' (but 'YS' is preferred for year-start) + if period.endswith('A') and not period.endswith('BA'): + if not _warned_a: + warnings.warn( + "'A' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version.", + DeprecationWarning, + stacklevel=3 + ) + _warned_a = True + if pandas_version < (2, 2): + return period[:-1] + 'YS' + + # Handle deprecated 'AS' -> 'YS' + if period.endswith('AS'): + if not _warned_as: + warnings.warn( + "'AS' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'YS' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_as = True + if pandas_version < (2, 2): + return period[:-2] + 'YS' + + # Handle deprecated 'H' -> 'h' + if period.endswith('H'): + if not _warned_h: + warnings.warn( + "'H' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'h' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_h = True + if pandas_version < (2, 2): + return period[:-1] + 'h' + + # Handle deprecated 'T' -> 'min' + if period.endswith('T'): + if not _warned_t: + warnings.warn( + "'T' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'min' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_t = True + if pandas_version < (2, 2): + return period[:-1] + 'min' + + # Handle deprecated 'S' -> 's' (but not 'MS', 'YS', 'AS', etc.) + if period.endswith('S') and not any(period.endswith(x) for x in ['MS', 'YS', 'AS', 'NS']): + if not _warned_s: + warnings.warn( + "'S' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 's' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_s = True + if pandas_version < (2, 2): + return period[:-1] + 's' + + return period + + def _freq_str_to_dateoffset(period): """ Convert a pandas frequency string to a pd.DateOffset. @@ -44,38 +164,96 @@ def _freq_str_to_dateoffset(period): :return: A pd.DateOffset :rtype: pd.DateOffset """ + global _warned_a + global _warned_as + global _warned_h + global _warned_t + global _warned_s + if period[-1] == 'M': as_dateoffset = pd.DateOffset(months=int(period[:-1])) elif period[-2:] == 'MS': as_dateoffset = pd.DateOffset(months=int(period[:-2])) elif period[-1] == 'A': + if not _warned_a: + warnings.warn( + "'A' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version.", + DeprecationWarning, + stacklevel=3 + ) + _warned_a = True as_dateoffset = pd.DateOffset(years=float(period[:-1])) elif period[-2:] == 'AS': + if not _warned_as: + warnings.warn( + "'AS' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'YS' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_as = True + as_dateoffset = pd.DateOffset(years=float(period[:-2])) + elif period[-2:] == 'YS': as_dateoffset = pd.DateOffset(years=float(period[:-2])) elif period[-1:] == 'W': as_dateoffset = pd.DateOffset(weeks=float(period[:-1])) elif period[-1:] == 'D': as_dateoffset = pd.DateOffset(days=float(period[:-1])) elif period[-1:] == 'H': + if not _warned_h: + warnings.warn( + "'H' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'h' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_h = True + as_dateoffset = pd.DateOffset(hours=float(period[:-1])) + elif period[-1:] == 'h': as_dateoffset = pd.DateOffset(hours=float(period[:-1])) elif period[-1:] == 'T': + if not _warned_t: + warnings.warn( + "'T' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 'min' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_t = True as_dateoffset = pd.DateOffset(minutes=float(period[:-1])) elif period[-3:] == 'min': as_dateoffset = pd.DateOffset(minutes=float(period[:-3])) elif period[-1:] == 'S': + if not _warned_s: + warnings.warn( + "'S' frequency string is deprecated by Pandas v2.2.0 and will be removed in a future " + "brightwind version. " + "Please use 's' instead.", + DeprecationWarning, + stacklevel=3 + ) + _warned_s = True + as_dateoffset = pd.DateOffset(seconds=float(period[:-1])) + elif period[-1:] == 's': as_dateoffset = pd.DateOffset(seconds=float(period[:-1])) else: - raise ValueError('"{}" period not recognized. Only units "M", "MS", "A", "AS", "W", "D", "H", "T", "min", "S" ' + raise ValueError('"{}" period not recognized. Only units "M", "MS", "YS", "W", "D", "h", "min", "s" ' 'are recognized'.format(period)) + + return as_dateoffset def _convert_days_to_hours(prd): - return str(int(prd[:-1])*24)+'H' + return str(int(prd[:-1])*24)+'h' def _convert_weeks_to_hours(prd): - return str(int(prd[:-1])*24*7)+'H' + return str(int(prd[:-1])*24*7)+'h' def _get_min_overlap_timestamp(df1_timestamps, df2_timestamps): @@ -174,7 +352,7 @@ def _round_timestamp_down_to_averaging_prd(timestamp, period): :param timestamp: Timestamp to round down from. :type timestamp: pd.Timestamp - :param period: Averaging period e.g. '10min', '1H', '3H', '6H', '1D', '7D', '1W', '1MS', '1AS' + :param period: Averaging period e.g. '10min', '1h', '3h', '6h', '1D', '7D', '1W', '1MS', '1YS' :type period: str :return: Timestamp to represent the start of an averaging period which covers the timestamp. :rtype: str @@ -188,12 +366,13 @@ def _round_timestamp_down_to_averaging_prd(timestamp, period): if 1M, 1MS it should go to start of month if 1A, 1AS it should go to start of year """ + period = _normalize_freq_string(period) if period[-3:] == 'min': return '{year}-{month}-{day} {hour}:{minute}:00'.format(year=timestamp.year, month=timestamp.month, day=timestamp.day, hour=timestamp.hour, minute=_round_down_to_multiple(timestamp.minute, int(period[:-3]))) - elif period[-1] == 'H': + elif period[-1] == 'h' or period[-1] == 'H': return '{year}-{month}-{day} {hour}:00:00'.format(year=timestamp.year, month=timestamp.month, day=timestamp.day, hour=_round_down_to_multiple(timestamp.hour, int(period[:-1]))) @@ -202,7 +381,7 @@ def _round_timestamp_down_to_averaging_prd(timestamp, period): hour=timestamp.hour) elif period[-1] == 'M' or period[-2:] == 'MS': return '{year}-{month}'.format(year=timestamp.year, month=timestamp.month) - elif period[-2:] == 'AS' or period[-1:] == 'A': + elif period[-2:] == 'YS' or period[-1:] == 'A' or period[-2:] == 'AS': return '{year}'.format(year=timestamp.year) else: print("Warning: Averaging period not identified returning default timestamps") @@ -337,11 +516,11 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met :param period: Groups data by the period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type period: str :param wdir_column_names: List of wind direction column names. These columns, if the aggregation_method is mean, @@ -376,7 +555,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met data = bw.load_csv(bw.demo_datasets.demo_data) # To find hourly averages - data_hourly = bw.average_data_by_period(data.Spd80mN, period='1H') + data_hourly = bw.average_data_by_period(data.Spd80mN, period='1h') # To find monthly averages data_monthly = bw.average_data_by_period(data.Spd80mN, period='1M') @@ -397,6 +576,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met """ coverage_threshold = validate_coverage_threshold(coverage_threshold) + period = _normalize_freq_string(period) if isinstance(period, str): if period[-1] == 'D': @@ -408,7 +588,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met if period[-1] == 'A': period = period+'S' if period[-1] == 'Y': - raise TypeError("Please use '1AS' for annual frequency at the start of the year.") + raise TypeError("Please use '1YS' for annual frequency at the start of the year.") # Check that the data resolution is not less than the period specified if data_resolution is None: @@ -416,8 +596,7 @@ def average_data_by_period(data, period, wdir_column_names=None, aggregation_met raise ValueError("The time period specified is less than the temporal resolution of the data. " "For example, hourly data should not be averaged to 10 minute data.") data = data.sort_index() - grouper_obj = data.resample(period, axis=0, closed='left', label='left', - convention='start', kind='timestamp') + grouper_obj = data.resample(period, closed='left', label='left') # if period is equal to data resolution then no need to vector average wind direction is_period_not_equal_to_resolution = (_freq_str_to_dateoffset(period) != _get_data_resolution(data.index)) @@ -542,7 +721,7 @@ def _vector_avg_of_wdirs_dataframe(wdirs, wspds=None): # means there is no wind direction => return NaN nan_mask = (avg_dir_df['sine'] == 0) & (avg_dir_df['cosine'] == 0) avg_dir_df['avg_dir'] = np.rad2deg(np.arctan2(sine, cosine)) % 360 - avg_dir_df['avg_dir'][nan_mask] = np.nan + avg_dir_df.loc[nan_mask,'avg_dir'] = np.nan return avg_dir_df['avg_dir'] @@ -705,11 +884,11 @@ def merge_datasets_by_period(data_1, data_2, period, :param period: Groups data by the time period specified here. The following formats are supported - Set period to '10min' for 10 minute average, '30min' for 30 minute average. - - Set period to '1H' for hourly average, '3H' for three hourly average and so on for '4H', '6H' etc. + - Set period to '1h' for hourly average, '3h' for three hourly average and so on for '4h', '6h' etc. - Set period to '1D' for a daily average, '3D' for three day average, similarly '5D', '7D', '15D' etc. - Set period to '1W' for a weekly average, '3W' for three week average, similarly '2W', '4W' etc. - Set period to '1M' for monthly average with the timestamp at the start of the month. - - Set period to '1A' for annual average with the timestamp at the start of the year. + - Set period to '1YS' for annual average with the timestamp at the start of the year. :type period: str :param wdir_column_names_1: List of wind direction column names. These columns, if the aggregation_method is mean, @@ -1058,7 +1237,7 @@ def offset_wind_direction(wdir, offset: float): if isinstance(wdir, float) or isinstance(wdir, int): return utils._range_0_to_360(wdir + offset) elif isinstance(wdir, pd.DataFrame): - return wdir.add(offset).applymap(utils._range_0_to_360) + return dataframe_map(wdir.add(offset), utils._range_0_to_360) elif isinstance(wdir, pd.Series): return wdir.add(offset).apply(utils._range_0_to_360) @@ -1367,14 +1546,14 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals - Set offset to 10min to add 10 minutes to each timestamp, -10min to subtract 10 minutes and so on for 4min, 20min, etc. - - Set offset to 1H to add 1 hour to each timestamp and -1H to subtract and so on for 5H, 6H, + - Set offset to 1h to add 1 hour to each timestamp and -1h to subtract and so on for 5h, 6h, etc. - Set offset to 1D to add a day and -1D to subtract and so on for 5D, 7D, 15D, etc. - Set offset to 1W to add a week and -1W to subtract from each timestamp and so on for 2W, 4W, etc. - Set offset to 1M to add a month and -1M to subtract a month from each timestamp and so on for 2M, 3M, etc. - - Set offset to 1Y to add an year and -1Y to subtract an year from each timestamp and so on + - Set offset to 1Y to add an year and -1Y to subtract a year from each timestamp and so on for 2Y, 3Y, etc. :type offset: str @@ -1402,7 +1581,7 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals data = bw.load_csv(bw.demo_datasets.demo_data) # To decrease 10 minutes within a given date range and overwrite the original data - op1 = bw.offset_timestamps(data, offset='1H', date_from='2016-02-01 00:20:00', + op1 = bw.offset_timestamps(data, offset='1h', date_from='2016-02-01 00:20:00', date_to='2016-02-01 01:40:00', overwrite=True) # To decrease 10 minutes within a given date range not overwriting the original data @@ -1419,14 +1598,14 @@ def offset_timestamps(data, offset, date_from=None, date_to=None, overwrite=Fals op4 = bw.offset_timestamps(data.index, offset='-10min', date_from='2016-02-01 00:20:00', date_to='2016-02-01 01:40:00') - # Can also except decimal values for offset, like 3.5H for 3 hours and 30 minutes - op5 = bw.offset_timestamps(data.index, offset='3.5H', date_from='2016-02-01 00:20:00', + # Can also except decimal values for offset, like 3.5h for 3 hours and 30 minutes + op5 = bw.offset_timestamps(data.index, offset='3.5h', date_from='2016-02-01 00:20:00', date_to='2016-02-01 01:40:00') # Can accept also Timestamp and datetime objects - bw.offset_timestamps(data.index[0], offset='4H') - bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5H') - bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5H') + bw.offset_timestamps(data.index[0], offset='4h') + bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5h') + bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5h') bw.offset_timestamps(datetime.time(0, 20), offset='30min') """ diff --git a/docs/source/tutorials/how_to_get_some_useful_stats.ipynb b/docs/source/tutorials/how_to_get_some_useful_stats.ipynb index 45915301..0688df2d 100644 --- a/docs/source/tutorials/how_to_get_some_useful_stats.ipynb +++ b/docs/source/tutorials/how_to_get_some_useful_stats.ipynb @@ -1877,7 +1877,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "But what if we dont want monthly coverage? We can then use the period variable to return whatever time period we want, whether that is 10-min (period='10min'), hourly (period='1H'), daily (period='1D'), weekly (period='1W') or yearly (period='1AS'). Here we have opted to return the yearly coverage." + "But what if we dont want monthly coverage? We can then use the period variable to return whatever time period we want, whether that is 10-min (period='10min'), hourly (period='1h'), daily (period='1D'), weekly (period='1W') or yearly (period='1YS'). Here we have opted to return the yearly coverage." ] }, { @@ -1964,7 +1964,7 @@ } ], "source": [ - "bw.coverage(data[anemometers],period='1AS')" + "bw.coverage(data[anemometers],period='1YS')" ] }, { diff --git a/docs/source/tutorials/how_to_transform_your_data.ipynb b/docs/source/tutorials/how_to_transform_your_data.ipynb index 39c18960..d45985c0 100644 --- a/docs/source/tutorials/how_to_transform_your_data.ipynb +++ b/docs/source/tutorials/how_to_transform_your_data.ipynb @@ -685,7 +685,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This function can be used to apply a window average to data, using a window which can be specified as any number of minutes ('min'), hours ('H'), days ('D'), weeks ('W'). '1M' and '1AS' can also used for monthly and annual averages respectively. The output is created within a new series. " + "This function can be used to apply a window average to data, using a window which can be specified as any number of minutes ('min'), hours ('h'), days ('D'), weeks ('W'). '1M' and '1YS' can also used for monthly and annual averages respectively. The output is created within a new series. " ] }, { @@ -1013,7 +1013,7 @@ ], "source": [ "# add 90 minutes to timestamps\n", - "data = bw.offset_timestamps(data, '1.5H')\n", + "data = bw.offset_timestamps(data, '1.5h')\n", "\n", "# print first timestamp\n", "print(data.index[0])" @@ -1041,7 +1041,7 @@ ], "source": [ "# subtract 2 hours from timestamps\n", - "data = bw.offset_timestamps(data, '-2H')\n", + "data = bw.offset_timestamps(data, '-2h')\n", "\n", "# print first timestamp\n", "print(data.index[0])" @@ -1332,7 +1332,7 @@ "outputs": [], "source": [ "# create new dataset with 6H added to the original timestamps\n", - "data_adj = bw.offset_timestamps(data, '6H')" + "data_adj = bw.offset_timestamps(data, '6h')" ] }, { diff --git a/setup.py b/setup.py index 595c573c..7e1c8e6d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def get_version(rel_path): 'gmaps>=0.9.0', 'colormap>=1.0.1', 'easydev>=0.10.0', - 'jsonschema>=4.17.3' + 'jsonschema>=4.17.3', + 'jinja2>= 3.0.0' ], classifiers=[ "Programming Language :: Python :: 3.6", diff --git a/tests/test_analyse.py b/tests/test_analyse.py index bc1f5159..1dde3149 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -139,9 +139,9 @@ def test_sector_ratio(): def test_basic_stats(): bw.basic_stats(DATA) bs2 = bw.basic_stats(DATA['Spd80mN']) - assert (bs2['count'][0] == 95180.0) and (round(bs2['mean'][0], 6) == 7.518636) and \ - (round(bs2['std'][0], 6) == 3.994552) and (round(bs2['min'][0], 3) == 0.215) and \ - (round(bs2['max'][0], 1) == 29.0) + assert (bs2['count'].iloc[0] == 95180.0) and (round(bs2['mean'].iloc[0], 6) == 7.518636) and \ + (round(bs2['std'].iloc[0], 6) == 3.994552) and (round(bs2['min'].iloc[0], 3) == 0.215) and \ + (round(bs2['max'].iloc[0], 1) == 29.0) def test_time_continuity_gaps(): @@ -209,9 +209,9 @@ def test_ti_twelve_by_24(): def test_coverage(): # hourly coverage - assert round(bw.coverage(DATA[['Spd80mN']], period='1H')[ + assert round(bw.coverage(DATA[['Spd80mN']], period='1h')[ '2016-01-09 17:00':'2016-01-09 17:30'].values[0][0], 5) == 0.83333 - assert round(bw.coverage(DATA.Spd80mN, period='1H')[ + assert round(bw.coverage(DATA.Spd80mN, period='1h')[ '2016-01-09 17:00':'2016-01-09 17:30'].values[0], 5) == 0.83333 # monthly_coverage assert round(bw.coverage(DATA.Spd80mN, period='1M')['2016-05-01'], 5) == 0.36537 diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 62964252..1708168a 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -102,7 +102,7 @@ def test_ordinary_least_squares(): assert round(correl.params['num_data_points'], 5) == correl_monthly_results_90_intercept['num_data_points'] # check hourly, checked against Excel - correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1H', + correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1h', coverage_threshold=1) correl.run() assert round(correl.params['slope'], 5) == correl_hourly_results['slope'] @@ -113,7 +113,7 @@ def test_ordinary_least_squares(): # check aggregation method correl_aggregation_results = {'slope': 5.98789, 'offset': -9.32585, 'r2': 0.9304, 'num_data_points': 12445} correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['T2M_degC'], DATA_CLND['T2m'], - averaging_prd='1H', coverage_threshold=1, + averaging_prd='1h', coverage_threshold=1, ref_aggregation_method='sum', target_aggregation_method='sum') correl.run() correl.plot() @@ -277,7 +277,7 @@ def test_synthesize(): for idx, row in pd.DataFrame(result_ord_lst_sq).iterrows(): # Comparing the first 6 digits to avoid issuing with floating point precision - assert str(row[0])[0:6] == str(synth.loc[idx][0])[0:6] + assert str(row.iloc[0])[0:6] == str(synth.loc[idx].iloc[0])[0:6] # Test the synthesise for when the ref_dir is given as input. correl = bw.Correl.OrdinaryLeastSquares(MERRA2_NE['WS50m_m/s']['2016-03-02 00:00:00':], @@ -288,7 +288,7 @@ def test_synthesize(): synth = correl.synthesize() for idx, row in pd.DataFrame(result_ord_lst_sq_dir).iterrows(): - assert str(row[0]) == str(round(synth.loc[idx][0], 6)) + assert str(row.iloc[0]) == str(round(synth.loc[idx].iloc[0], 6)) # Test the synthesise when SpeedSort correlation is used. correl = bw.Correl.SpeedSort(MERRA2_NE['WS50m_m/s']['2016-03-02 00:00:00':'2017-03-02 00:00:00'], @@ -301,14 +301,15 @@ def test_synthesize(): for idx, row in pd.DataFrame(result_speed_sort).iterrows(): print(idx) - assert str(row[0]) == str(round(synth.loc[idx][0], 6)) + assert str(row.iloc[0]) == str(round(synth.loc[idx].iloc[0], 6)) # Test the synthesise when SpeedSort correlation is used using 10 min averaging period. data_test = DATA_CLND[['Spd80mN', 'Spd60mN', 'Dir78mS', 'Dir58mS']].copy() - data_test['Dir78mS']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan - data_test['Spd80mN']['2016-01-09 17:10:00':'2016-01-09 17:50:00'] = np.nan - data_test['Dir58mS']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan - data_test['Spd60mN']['2016-01-09 17:50:00':'2016-01-10 19:10:00'] = np.nan + data_test.loc['2016-01-09 17:10:00':'2016-01-09 17:50:00', 'Dir78mS'] = np.nan + data_test.loc['2016-01-09 17:10:00':'2016-01-09 17:50:00', 'Spd80mN'] = np.nan + data_test.loc['2016-01-09 17:50:00':'2016-01-10 19:10:00', 'Dir58mS'] = np.nan + data_test.loc['2016-01-09 17:50:00':'2016-01-10 19:10:00', 'Spd60mN'] = np.nan + ss_cor = bw.Correl.SpeedSort(data_test['Spd80mN'], data_test['Dir78mS'], data_test['Spd60mN'], data_test['Dir58mS'], averaging_prd='10min') ss_cor.run() @@ -370,7 +371,7 @@ def test_orthogonal_least_squares(): assert round(correl.params['num_data_points'], 5) == correl_monthly_results['num_data_points'] # check hourly - correl = bw.Correl.OrthogonalLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1H', + correl = bw.Correl.OrthogonalLeastSquares(MERRA2_NE['WS50m_m/s'], DATA_CLND['Spd80mN'], averaging_prd='1h', coverage_threshold=1) correl.run() assert round(correl.params['slope'], 5) == correl_hourly_results['slope'] @@ -381,7 +382,7 @@ def test_orthogonal_least_squares(): # check aggregation method correl_aggregation_results = {'slope': 6.42434, 'offset': -12.8301, 'r2': 0.9255, 'num_data_points': 12445} correl = bw.Correl.OrthogonalLeastSquares(MERRA2_NE['T2M_degC'], DATA_CLND['T2m'], - averaging_prd='1H', coverage_threshold=1, + averaging_prd='1h', coverage_threshold=1, ref_aggregation_method='sum', target_aggregation_method='sum') correl.run() assert round(correl.params['slope'], 5) == correl_aggregation_results['slope'] @@ -403,7 +404,7 @@ def test_multiple_linear_regression(): # check aggregation method correl_aggregation_results = {'slope': [5.51666, 0.54769], 'offset': -10.44818, 'num_data_pts': 12445} correl = bw.Correl.MultipleLinearRegression([MERRA2_NE['T2M_degC'], MERRA2_NW['T2M_degC']], DATA_CLND['T2m'], - averaging_prd='1H', coverage_threshold=1, + averaging_prd='1h', coverage_threshold=1, ref_aggregation_method='sum', target_aggregation_method='sum') correl.run() for idx, slope in enumerate(correl.params['slope']): @@ -570,7 +571,7 @@ def test_speed_sort(): } ss_cor = bw.Correl.SpeedSort(MERRA2_NE['WS50m_m/s'], MERRA2_NE['WD50m_deg'], DATA_CLND['Spd80mN'], DATA_CLND['Dir78mS'], - averaging_prd='1H') + averaging_prd='1h') ss_cor.run() assert ss_cor.params['overall_average_veer'] == result['overall_average_veer'] diff --git a/tests/test_shear.py b/tests/test_shear.py index 39a284d8..bf083c69 100644 --- a/tests/test_shear.py +++ b/tests/test_shear.py @@ -136,8 +136,8 @@ def test_time_of_day(): # Test attributes - assert round(shear_by_tod_power_law2.alpha.mean()[0], 4) == 0.1473 - assert round(shear_by_tod_log_law2.roughness.mean()[0], 4) == 0.1450 + assert round(shear_by_tod_power_law2.alpha.mean().iloc[0], 4) == 0.1473 + assert round(shear_by_tod_log_law2.roughness.mean().iloc[0], 4) == 0.1450 # Test apply assert (round(DATA['Spd80mN']['2017-11-23 10:10:00':'2017-11-23 10:40:00'] * ( @@ -221,12 +221,12 @@ def test_time_series(): shear_by_ts_power_law.apply(DATA['Spd80mN'], 40, 60) shear_by_ts_log_law.apply(DATA['Spd80mN'], 40, 60) - DATA['Spd80mN'].iloc[0] = 2 + DATA.loc[DATA.index[0], 'Spd80mN'] = 2 shear_ts = bw.Shear.TimeSeries(anemometers, heights) alpha = shear_ts.alpha assert pd.isna(alpha.iloc[0]) - DATA['Spd80mN'].iloc[0] = np.nan + DATA.loc[DATA.index[0], 'Spd80mN'] = np.nan shear_ts = bw.Shear.TimeSeries(anemometers, heights, calc_method='log_law') roughness = shear_ts.roughness assert pd.isna(roughness.iloc[0]) diff --git a/tests/test_transform.py b/tests/test_transform.py index 932442d5..fe9b1c2b 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -7,6 +7,7 @@ import io import sys import re +import warnings from brightwind.transform.transform import ( _check_vertical_profiler_properties_overlap as check_vertical_profiler_properties_overlap @@ -292,13 +293,13 @@ def test_check_vertical_profiler_properties_not_overlap(): def test_freq_str_to_dateoffset(): # Excluding monthly periods and above as it will depend on which month or year periods = ['1S', '1min', '5min', '10min', '15min', - '1H', '3H', '6H', '1D', '7D', + '1h', '3h', '6h', '1D', '7D', '1W', '2W', '1MS', '1M', '3M', '6MS', - '1AS', '1A', '3A'] + '1YS'] results = [1.0, 60.0, 300.0, 600.0, 900.0, 3600.0, 10800.0, 21600.0, 86400.0, 604800.0, 604800.0, 1209600.0, 2678400.0, 2678400.0, 7862400.0, 15724800.0, - 31622400.0, 31622400.0, 94694400.0] + 31622400.0] for idx, period in enumerate(periods): if type(bw.transform.transform._freq_str_to_dateoffset(period)) == pd.DateOffset: @@ -308,18 +309,62 @@ def test_freq_str_to_dateoffset(): ).total_seconds() == results[idx] # Check that data frequency is returned as a DateOffset. - assert type(bw.transform.transform._freq_str_to_dateoffset(period)) == pd.DateOffset + assert isinstance(bw.transform.transform._freq_str_to_dateoffset(period), pd.DateOffset) + + +def test_freq_str_to_dateoffset_deprecation_warning(): + """Test frequency string conversion and check deprecation warnings for old formats.""" + ref_date = pd.Timestamp('2020-01-01') + + periods = [ + '1H', '3H', '6H','1T', '30T', '1S', '1AS', '1A', '3A' + ] + results = [ + 3600.0, 10800.0, 21600.0, 60.0, 1800.0, 1.0, 31622400.0, 31622400.0, 94694400.0 + ] + + # Reset warning flags to test them + bw.transform.transform._warned_h = False + bw.transform.transform._warned_t = False + bw.transform.transform._warned_s = False + bw.transform.transform._warned_a = False + bw.transform.transform._warned_as = False + + for idx, period in enumerate(periods): + # All these deprecated formats should raise DeprecationWarning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = bw.transform.transform._freq_str_to_dateoffset(period) + + # Check warning was raised on first use of each suffix + if period in ['1H', '1T', '1S', '1AS', '1A']: + assert len(w) == 1, f"Expected warning for deprecated format '{period}'" + assert issubclass(w[0].category, DeprecationWarning) + + # Check that data frequency is returned as a DateOffset + assert isinstance(result, pd.DateOffset) + + # Verify the time delta + if type(result) == pd.DateOffset: + assert (ref_date + result - ref_date).total_seconds() == results[idx] def test_round_timestamp_down_to_averaging_prd(): timestamp = pd.Timestamp('2016-01-09 11:21:11') - avg_periods = ['10min', '15min', '1H', '3H', '6H', '1D', '7D', '1W', '1MS', '1AS'] + avg_periods = ['10min', '15min', '1h', '3h', '6h', '1D', '7D', '1W', '1MS', '1YS'] avg_period_start_timestamps = ['2016-1-9 11:20:00', '2016-1-9 11:15:00', '2016-1-9 11:00:00', '2016-1-9 9:00:00', '2016-1-9 6:00:00', '2016-1-9', '2016-1-9', '2016-1-9', '2016-1', '2016'] for idx, avg_period in enumerate(avg_periods): assert avg_period_start_timestamps[idx] == \ bw.transform.transform._round_timestamp_down_to_averaging_prd(timestamp, avg_period) + avg_periods = ['1H', '3H', '6H', '1AS'] + avg_period_start_timestamps = [ + '2016-1-9 11:00:00', '2016-1-9 9:00:00', '2016-1-9 6:00:00', '2016' + ] + for idx, avg_period in enumerate(avg_periods): + assert avg_period_start_timestamps[idx] == \ + bw.transform.transform._round_timestamp_down_to_averaging_prd(timestamp, avg_period) def test_get_data_resolution(): @@ -328,7 +373,7 @@ def test_get_data_resolution(): series1 = DATA['Spd80mS'].index assert bw.transform.transform._get_data_resolution(series1).kwds == {'minutes': 10} - series2 = pd.date_range('2010-01-01', periods=150, freq='H') + series2 = pd.date_range('2010-01-01', periods=150, freq='h') assert bw.transform.transform._get_data_resolution(series2).kwds == {'hours': 1} series2 = pd.date_range('2010-01-01', periods=150, freq='D') @@ -340,9 +385,19 @@ def test_get_data_resolution(): series1 = bw.average_data_by_period(DATA['Spd80mN'], period='1M', coverage_threshold=0, return_coverage=False) assert bw.transform.transform._get_data_resolution(series1.index).kwds == {'months': 1} + series1 = bw.average_data_by_period(DATA['Spd80mN'], period='1YS', coverage_threshold=0, return_coverage=False) + assert bw.transform.transform._get_data_resolution(series1.index).kwds == {'years': 1} + series1 = bw.average_data_by_period(DATA['Spd80mN'], period='1AS', coverage_threshold=0, return_coverage=False) assert bw.transform.transform._get_data_resolution(series1.index).kwds == {'years': 1} + # hourly series with one instance where difference between adjacent timestamps is 10 min + series3 = pd.date_range('2010-04-15', '2010-05-01', freq='h').union(pd.date_range('2010-05-01 00:10:00', periods=20, + freq='h')) + with warnings.catch_warnings(record=True) as w: + assert bw.transform.transform._get_data_resolution(series3).kwds == {'hours': 1} + assert len(w) == 1 + # hourly series with one instance where difference between adjacent timestamps is 10 min series3 = pd.date_range('2010-04-15', '2010-05-01', freq='H').union(pd.date_range('2010-05-01 00:10:00', periods=20, freq='H')) @@ -449,9 +504,14 @@ def test_offset_timestamps(): assert (op.loc['2016-01-12 00:00:00'] == series1.Spd60mN.loc['2016-01-11 23:30:00']).all() assert (op.loc['2016-01-12 00:30:00'] == series1.Spd60mN.loc['2016-01-12 00:30:00']).all() + assert bw.offset_timestamps(DATA.index[0], offset='4h') == pd.Timestamp('2016-01-09 19:30:00') + assert bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5h' + ) == datetime.datetime(2016, 2, 1, 3, 50) + assert bw.offset_timestamps(DATA.index[0], offset='4H') == pd.Timestamp('2016-01-09 19:30:00') assert bw.offset_timestamps(datetime.datetime(2016, 2, 1, 0, 20), offset='3.5H' ) == datetime.datetime(2016, 2, 1, 3, 50) + assert bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5h') == datetime.datetime(2016, 1, 31, 19, 0) assert bw.offset_timestamps(datetime.date(2016, 2, 1), offset='-5H') == datetime.datetime(2016, 1, 31, 19, 0) assert bw.offset_timestamps(datetime.time(0, 20), offset='30min') == datetime.time(0, 50) @@ -541,46 +601,48 @@ def test_average_wdirs(): for i, j in zip(avg_wdirs, expected_result): assert i == j - -def dummy_data_frame(start_date='2016-01-01T00:00:00', end_date='2016-12-31T11:59:59'): +@pytest.fixture +def dummy_data(): """ - Returns a DataFrame with wind speed equal to the month of the year, i.e. In January, wind speed = 1 m/s. - For use in testing. - - :param start_date: Start date Timestamp, i.e. first index in the DataFrame - :type start_date: Timestamp as a string in the form YYYY-MM-DDTHH:MM:SS' - :param end_date: End date Timestamp, i.e. last index in the DataFrame - :type end_date: Timestamp as a string in the form YYYY-MM-DDTHH:MM:SS' - :return: pandas.DataFrame + Fixture that returns a DataFrame with wind speed equal to the month of the year. + Wind speed in January = 1 m/s, February = 2 m/s, etc. """ - - date_times = {'Timestamp': pd.date_range(start_date, end_date, freq='10T')} - - dummy_wind_speeds = [] - dummy_wdirs = [] - - for i, vals in enumerate(date_times['Timestamp']): - # get list of each month for each date entry as dummy windspeeds - dummy_wind_speeds.append(vals.month) - dummy_wdirs.append((vals.month - 1) * 30) - - dummy_wind_speeds_df = pd.DataFrame({'wspd': dummy_wind_speeds, 'wdir': dummy_wdirs}, index=date_times['Timestamp']) + start_date = '2016-01-01T00:00:00' + end_date = '2016-12-31T11:59:59' + + date_times = pd.date_range(start_date, end_date, freq='10min') + + # Vectorized approach - compatible with all pandas versions + dummy_wind_speeds = date_times.month.values + dummy_wdirs = (date_times.month.values - 1) * 30 + + dummy_wind_speeds_df = pd.DataFrame( + {'wspd': dummy_wind_speeds, 'wdir': dummy_wdirs}, + index=date_times + ) dummy_wind_speeds_df.index.name = 'Timestamp' - + return dummy_wind_speeds_df -def test_average_data_by_period(): +def test_average_data_by_period(dummy_data): + bw.average_data_by_period(DATA[['Spd80mN']], period='1h') bw.average_data_by_period(DATA[['Spd80mN']], period='1H') # hourly averages + bw.average_data_by_period(DATA.Spd80mN, period='1h') bw.average_data_by_period(DATA.Spd80mN, period='1H') # hourly average with coverage filtering + bw.average_data_by_period(DATA.Spd80mN, period='1h', coverage_threshold=0.9) + bw.average_data_by_period(DATA.Spd80mN, period='1h', coverage_threshold=1) bw.average_data_by_period(DATA.Spd80mN, period='1H', coverage_threshold=0.9) bw.average_data_by_period(DATA.Spd80mN, period='1H', coverage_threshold=1) # return coverage with filtering + bw.average_data_by_period(DATA.Spd80mN, period='1h', coverage_threshold=0.9, + return_coverage=True) bw.average_data_by_period(DATA.Spd80mN, period='1H', coverage_threshold=0.9, return_coverage=True) # return coverage without filtering + bw.average_data_by_period(DATA.Spd80mN, period='1h', return_coverage=True) bw.average_data_by_period(DATA.Spd80mN, period='1H', return_coverage=True) # monthly averages @@ -608,7 +670,6 @@ def test_average_data_by_period(): assert str(except_info.value) == "The time period specified is less than the temporal resolution of the data. " \ "For example, hourly data should not be averaged to 10 minute data." - dummy_data = dummy_data_frame() average_monthly_speed = bw.average_data_by_period(dummy_data.wspd, period='1M') # test average wind speed for each month for i in range(0, 11): @@ -641,6 +702,8 @@ def test_average_data_by_period(): assert average_monthly_speed[1].count().wspd_Coverage == 12 # the returned coverage has 12 months # test average annual wind speed + average_annual_speed = bw.average_data_by_period(dummy_data.wspd, period='1YS') + assert round(average_annual_speed.iloc[0].item(), 3) == 6.506 average_annual_speed = bw.average_data_by_period(dummy_data.wspd, period='1AS') assert round(average_annual_speed.iloc[0].item(), 3) == 6.506 # average DATA to monthly @@ -686,8 +749,7 @@ def test_average_data_by_period(): data_monthly, coverage_monthly = bw.average_data_by_period(data_test, period='1M', wdir_column_names='Dir78mS', return_coverage=True, data_resolution=pd.DateOffset(minutes=10)) - table_count = data_test.resample('1MS', axis=0, closed='left', label='left', - convention='start', kind='timestamp').count() + table_count = data_test.resample('1MS', closed='left', label='left').count() assert (table_count['Dir78mS']['2016-01-01'] / (31 * 24 * 6) - coverage_monthly['Dir78mS_Coverage']['2016-01-01'] ) < 1e-5 assert (table_count['Spd80mN']['2016-01-01'] / (31 * 24 * 6) - coverage_monthly['Spd80mN_Coverage']['2016-01-01'] From 3c72734c30ce4222cfbd3ade12705b6bb56ad779 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Tue, 16 Dec 2025 13:06:29 +0000 Subject: [PATCH 24/27] iss #556 add deprecated warning to BrightData (#564) * iss #556 add deprecated warning to BrightData * iss #556 fix text * iss #556 added additional deprecated notices while i was at it --- CHANGELOG.md | 6 +++++- brightwind/load/load.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30ed7d16..cff8f6a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ Additional labels for pre-release and build metadata are available as extensions 1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) 1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) - ### Bug Fixes 1. @@ -34,6 +33,11 @@ Additional labels for pre-release and build metadata are available as extensions 1. Updated `time_continuity_gaps()` in order to take an argument `minimum_gap_length` which allows the user to filter the time gaps returned. (Issue [#545](https://github.com/brightwind-dev/brightwind/issues/545)) 1. Updated `offset_timestamps()`to include last timestamp when date_to is unspecified, so that offset is applied to the entire record if date_to not specified. ([#504](https://github.com/brightwind-dev/brightwind/issues/504)) +### Deprecated +1. `LoadBrightdata()` is deprecated and will be removed in version 3.0. Please use `LoadBrightHub()` instead to continue accessing reanalysis data. +2. `LoadBrightHub()` authentication using email and password is deprecated and will be removed in v3.0. Please migrate to API key authentication. Create and manage API keys at: https://brighthub.io/account-settings/settings. +3. `specific_gas_constant` argument of `calc_air_density()` is deprecated and will be removed in v3.0 because the updated method depends on the gas constants for dry air and water vapour, making this argument redundant. +4. The scaling of air density to height within `calc_air_density()` is deprecated and will be removed in v3.0. Users should use `scale_air_density_to_height()` separately instead. ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. diff --git a/brightwind/load/load.py b/brightwind/load/load.py index 7f02f700..bcf5a8e6 100644 --- a/brightwind/load/load.py +++ b/brightwind/load/load.py @@ -447,6 +447,12 @@ def _append_files_together(source_folder, assembled_file_name, file_type, append class LoadBrightdata: """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + LoadBrightdata allows you to pull meta data and timeseries data of reanalysis datasets from brightdata. This is a fast way to get access to the available reanalysis datasets. @@ -497,6 +503,15 @@ def _get_brightdata(sub_uri, **query_params): :param query_params: dictionary of the query parameters to be sent :return: List(Node) """ + + warnings.warn( + "BrightDATA was deprecated in v2.4.0 and will be removed in v3.0.0. " + "Support for reanalysis data has moved to BrightHub. " + "Please use LoadBrightHub instead of LoadBrightdata to continue accessing reanalysis data.", + DeprecationWarning, + stacklevel=3 + ) + username = utils.get_environment_variable('BRIGHTDATA_USERNAME') password = utils.get_environment_variable('BRIGHTDATA_PASSWORD') @@ -548,6 +563,12 @@ def _parse_variables(variables_list): @staticmethod def timeseries(dataset, lat, long, nearest=None, from_date=None, to_date=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Retrieve timeseries datasets available from brightdata. Returns a list of Node objects in order of closest distance to the requested lat, long. @@ -632,6 +653,12 @@ def timeseries(dataset, lat, long, nearest=None, from_date=None, to_date=None, v @staticmethod def monthly_means(dataset, lat, long, nearest=None, from_date=None, to_date=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Retrieve monthly means from brightdata datasets such as merra2 and era5. Returns a list of Node objects in order of closest distance to the requested lat, long. Monthly coverage is also returned in the data. @@ -713,6 +740,12 @@ def monthly_means(dataset, lat, long, nearest=None, from_date=None, to_date=None @staticmethod def momm(dataset, lat, long, nearest=None, from_date=None, to_date=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Retrieve the mean of monthly means from brightdata datasets such as merra2 and era5. Returns a list of Node objects in order of closest distance to the requested lat, long. @@ -796,6 +829,12 @@ def momm(dataset, lat, long, nearest=None, from_date=None, to_date=None, variabl def monthly_norms(dataset, lat, long, nearest=None, from_date=None, to_date=None, ref_from_date=None, ref_to_date=None, ref_no_years=None, variables=None): """ + ** BRIGHTDATA IS NO LONGER SUPPORTED WITH NEW DATA. THIS FUNCTION WAS DEPRECIATED IN V2.4.0 AND WILL BE + REMOVED IN V3.0. + + SUPPORT FOR REANALYSIS DATA HAS MOVED TO BRIGHTHUB. PLEASE USE LOADBRIGHTHUB INSTEAD OF LOADBRIGHTDATA TO + CONTINUE ACCESSING REANALYSIS DATA. ** + Return the monthly mean wind speeds normalised to a specific reference period. The reference period can be between two specific dates or it could be a number of rolling years preceding each month of interest. From 8955dae89dcdd6d66e35f46c851f20852e457cfd Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Tue, 16 Dec 2025 15:03:53 +0000 Subject: [PATCH 25/27] iss #561 renamed series with _scaled (#563) --- brightwind/transform/scale.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/brightwind/transform/scale.py b/brightwind/transform/scale.py index 7bf903ea..108026c6 100644 --- a/brightwind/transform/scale.py +++ b/brightwind/transform/scale.py @@ -221,18 +221,21 @@ def scale_air_density_to_height(ref_air_density_kg_m3: Union[float, pd.Series], bw.scale_air_density_to_height(ref_air_density_kg_m3=test_density.loc['2016-01-09 17:10':'2016-01-09 18:00'], ref_height_m=2, target_height_m=10) - Timestamp - 2016-01-09 17:10:00 1.186780 - 2016-01-09 17:20:00 1.187175 - 2016-01-09 17:30:00 1.187747 - 2016-01-09 17:40:00 1.185950 - 2016-01-09 17:50:00 1.186301 - 2016-01-09 18:00:00 1.185686 - dtype: float64 + # Timestamp + # 2016-01-09 17:10:00 1.186780 + # 2016-01-09 17:20:00 1.187175 + # 2016-01-09 17:30:00 1.187747 + # 2016-01-09 17:40:00 1.185950 + # 2016-01-09 17:50:00 1.186301 + # 2016-01-09 18:00:00 1.185686 + # Name: air_density_scaled, dtype: float64 """ scaled_air_density = linear_transform(x_target=target_height_m, x_ref=ref_height_m, y_ref=ref_air_density_kg_m3, slope=lapse_rate_kg_m3_m) + if isinstance(scaled_air_density, pd.Series): + air_density_series_name = ref_air_density_kg_m3.name if ref_air_density_kg_m3.name else "air_density" + scaled_air_density.name = f"{air_density_series_name}_scaled" return scaled_air_density @@ -298,11 +301,14 @@ def scale_air_temperature_to_height(ref_air_temperature: Union[float, pd.Series] # 2016-01-09 17:40:00 0.735 # 2016-01-09 17:50:00 0.654 # 2016-01-09 18:00:00 0.796 - # Name: T2m, dtype: float64 + # Name: T2m_scaled, dtype: float64 """ scaled_air_temp = linear_transform(x_target=target_height_m, x_ref=ref_height_m, y_ref=ref_air_temperature, slope=lapse_rate_deg_m) + if isinstance(scaled_air_temp, pd.Series): + air_temp_series_name = ref_air_temperature.name if ref_air_temperature.name else "air_temperature" + scaled_air_temp.name = f"{air_temp_series_name}_scaled" return scaled_air_temp @@ -360,7 +366,7 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], # 2016-01-09 17:40:00 932.07 # 2016-01-09 17:50:00 932.07 # 2016-01-09 18:00:00 932.07 - # dtype: float64 + # Name: P2m_scaled, dtype: float64 """ @@ -386,5 +392,7 @@ def scale_air_pressure_to_height(ref_air_pressure_hPa: Union[float, pd.Series], scaled_air_pressure_hPa = ref_air_pressure_hPa * ((1 + (L / ref_air_temp_K) * (target_height_m - ref_height_m) ) ** (-g / (L * R))) - + if isinstance(scaled_air_pressure_hPa, pd.Series): + air_pressure_series_name = ref_air_pressure_hPa.name if ref_air_pressure_hPa.name else "air_pressure" + scaled_air_pressure_hPa.name = f"{air_pressure_series_name}_scaled" return scaled_air_pressure_hPa From c6eaa43b7ef7ebdd869835f3650f985d17d2573c Mon Sep 17 00:00:00 2001 From: Sara <183604246+sararafter@users.noreply.github.com> Date: Wed, 17 Dec 2025 10:13:01 +0000 Subject: [PATCH 26/27] [Iss566] Handle dataframes in calc_air_density (#567) * iss #566 raise error if dataframe with more than one column provided * iss #566 remove dataframe type and edit tests --- brightwind/analyse/analyse.py | 26 +++++++++++++------------- tests/test_analyse.py | 10 ++-------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/brightwind/analyse/analyse.py b/brightwind/analyse/analyse.py index bce9e967..9c10a7a9 100644 --- a/brightwind/analyse/analyse.py +++ b/brightwind/analyse/analyse.py @@ -2042,13 +2042,13 @@ def sector_ratio(wspd_1, wspd_2, wdir, sectors=72, min_wspd=3, direction_bin_arr return fig -def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], - pressure: Union[float, pd.Series, pd.DataFrame], +def calc_air_density(temperature: Union[float, pd.Series], + pressure: Union[float, pd.Series], elevation_ref: Union[float, int] = None, elevation_site: Union[float, int] = None, lapse_rate: float = AIR_DENSITY_LAPSE_RATE, specific_gas_constant: float = 286.9, - rel_humidity_percent: Union[float, pd.Series, pd.DataFrame] = None + rel_humidity_percent: Union[float, pd.Series] = None ) -> Union[float, pd.Series]: """ Calculates air density for a given air temperature (temperature), air pressure (pressure) @@ -2072,10 +2072,10 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], `scale_air_density_to_height()` separately instead. :param temperature: Air temperature values in degree Celsius - :type temperature: float or pandas.Series or pandas.DataFrame + :type temperature: float or pandas.Series :param pressure: Air pressure values in hectopascal, hPa or millibar, mbar (Note 1hPa = 1mbar) (1013.25 hPa = 101,325 Pa = 101.325 kPa = 1 atm = 1013.25 mbar) - :type pressure: float or pandas.Series or pandas.DataFrame + :type pressure: float or pandas.Series :param elevation_ref: Elevation, in meters, of the reference air density location (and air temperature, air pressure, relative humidity measurements used for the calculation). :type elevation_ref: float or int @@ -2094,10 +2094,10 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], :type specific_gas_constant: float :param rel_humidity_percent: Relative humidity values as a percentage. Default is None. If None, the air density calculation ignores humidity. - :type rel_humidity_percent: float or pandas.Series or pandas.DataFrame + :type rel_humidity_percent: float or pandas.Series :return: Air density in kg/m^3. Output type depends on type(temperature), type(pressure), type(rel_humidity_percent) provided. If all inputs are float, output is float. - If any input is pandas.Series or pandas.DataFrame, output is pandas.Series. + If any input is pandas.Series, output is pandas.Series. :rtype: float or pandas.Series **Example usage** @@ -2160,18 +2160,18 @@ def calc_air_density(temperature: Union[float, pd.Series, pd.DataFrame], # 1.198 """ - # Convert DataFrame inputs to Series + # Raise errors if any inputs are DataFrames if isinstance(temperature, pd.DataFrame): - temperature = _convert_df_to_series(temperature).copy() + raise ValueError("temperature cannot be provided as a DataFrame. Provide as float or pandas Series.") if isinstance(pressure, pd.DataFrame): - pressure = _convert_df_to_series(pressure).copy() + raise ValueError("pressure cannot be provided as a DataFrame. Provide as float or pandas Series.") if isinstance(rel_humidity_percent, pd.DataFrame): - rel_humidity_percent = _convert_df_to_series(rel_humidity_percent).copy() - + raise ValueError("rel_humidity_percent cannot be provided as a DataFrame. Provide as float or pandas Series.") + # Check dimensions of temperature, pressure and rel_humidity_percent if not float or int if isinstance(temperature, pd.Series) and (isinstance(pressure, pd.Series)): if len(temperature) != len(pressure): - raise ValueError("temperature and pressure must have the same dimensions.") + raise ValueError("temperature and pressure must have the same dimensions.") if isinstance(rel_humidity_percent, pd.Series): if len(temperature) != len(rel_humidity_percent): raise ValueError("temperature, pressure and rel_humidity_percent must have the same dimensions.") diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 1dde3149..e61e1d3b 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -638,12 +638,6 @@ def test_ti_by_sector(): def test_calc_air_density(): - - # test DataFrame input - assert (bw.calc_air_density(DATA[['T2m']], DATA[['P2m']]).dropna().values == - bw.calc_air_density(DATA.T2m, DATA.P2m).dropna().values).all() - assert (round(bw.calc_air_density(pd.Series([15, 12.5, -5, 23]), pd.Series([1013, 990, 1020, 900])), 6 - ) == pd.Series([1.225350, 1.208010, 1.325842, 1.059254])).all() # test Series inputs assert list(round(bw.calc_air_density(DATA.T2m, DATA.P2m).tail(5), 6).values @@ -684,10 +678,10 @@ def test_calc_air_density(): bw.calc_air_density(15, 1013, elevation_ref=200) assert str(except_info.value) == "Specify value of elevation_site (float or int) when elevation_ref is provided." with pytest.raises(ValueError) as except_info: - bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) + bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(3), rel_humidity_percent=DATA.RH2m.tail(5)) assert str(except_info.value) == "temperature and pressure must have the same dimensions." with pytest.raises(ValueError) as except_info: - bw.calc_air_density(DATA.T2m.tail(5), DATA[['P2m']].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) + bw.calc_air_density(DATA.T2m.tail(5), DATA['P2m'].tail(5), rel_humidity_percent=DATA.RH2m.tail(3)) assert str(except_info.value) == "temperature, pressure and rel_humidity_percent must have the same dimensions." From a1713c61c977051f25b8c6d2dae72328d6c4c489 Mon Sep 17 00:00:00 2001 From: stephenholleran Date: Wed, 17 Dec 2025 10:27:59 +0000 Subject: [PATCH 27/27] prepare for v2.4.0 release (#565) --- CHANGELOG.md | 14 ++++++++------ brightwind/__init__.py | 23 ++++++++++++++++++++++- setup.py | 4 ++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cff8f6a6..7013f90c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,7 @@ Additional labels for pre-release and build metadata are available as extensions --- -## [2.4.0-dev] -1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) -1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) - -### Bug Fixes -1. +## [2.4.0] ### New Features and Enhancements 1. Added `scale_air_pressure_to_height()` to output an air pressure value for any height based on reference air temperature and air pressure values at a different measurement height. ([#531](https://github.com/brightwind-dev/brightwind/issues/531)) @@ -38,6 +33,13 @@ Additional labels for pre-release and build metadata are available as extensions 2. `LoadBrightHub()` authentication using email and password is deprecated and will be removed in v3.0. Please migrate to API key authentication. Create and manage API keys at: https://brighthub.io/account-settings/settings. 3. `specific_gas_constant` argument of `calc_air_density()` is deprecated and will be removed in v3.0 because the updated method depends on the gas constants for dry air and water vapour, making this argument redundant. 4. The scaling of air density to height within `calc_air_density()` is deprecated and will be removed in v3.0. Users should use `scale_air_density_to_height()` separately instead. +5. Support for **Python ≤ 3.10** is deprecated and will be dropped in v3.0.0. Users should upgrade to Python 3.11 or newer. +6. Support for **Pandas ≤ 2.2** is deprecated and will be dropped in v3.0.0. Users should upgrade to Pandas 2.3 or newer. + +### Bug Fixes +1. Fixed pandas<3.0.0 and numpy<2.3.1 dependencies, ([#458](https://github.com/brightwind-dev/brightwind/issues/458)) +1. Fixed pandas deprecating warnings that were linked to frequency strings, .groupby() and .map(). ([#407](https://github.com/brightwind-dev/brightwind/issues/407), [#415](https://github.com/brightwind-dev/brightwind/issues/415) and [#445](https://github.com/brightwind-dev/brightwind/issues/445)) + ## [2.3.0] This update brings a comprehensive set of **bug fixes** and **enhancements** across the Brightwind library. diff --git a/brightwind/__init__.py b/brightwind/__init__.py index 5814a0f6..d865d7f5 100644 --- a/brightwind/__init__.py +++ b/brightwind/__init__.py @@ -11,6 +11,27 @@ from .utils.gis import * # from .utils.utils import * +import sys +import warnings +from packaging.version import Version +import pandas as pd + +if sys.version_info < (3, 11): + warnings.warn( + "Support for Python 3.10 is deprecated and will be removed in v3.0.0. " + "Please upgrade to Python 3.11 or newer.", + FutureWarning, + stacklevel=2, + ) + +if Version(pd.__version__) <= Version("2.2"): + warnings.warn( + "Support for pandas ≤ 2.2 is deprecated and will be removed in v3.0.0. " + "Please upgrade to pandas 2.3 or newer.", + FutureWarning, + stacklevel=2, + ) + __all__ = ['analyse', 'transform', 'export', 'load', 'demo_datasets'] -__version__ = '2.4.0-dev' +__version__ = '2.4.0' diff --git a/setup.py b/setup.py index 7e1c8e6d..3c4fe9c7 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def get_version(rel_path): setup( name='brightwind', - # Update version number here, below and in _init_.py: + # Update version number below and in _init_.py: version=get_version("brightwind/__init__.py"), packages=['brightwind', 'brightwind.load', 'brightwind.utils', 'brightwind.export', 'brightwind.analyse', 'brightwind.transform', 'brightwind.demo_datasets'], @@ -34,7 +34,7 @@ def get_version(rel_path): }, url='https://github.com/brightwind-dev/brightwind.git', # UPDATE VERSION NUMBER HERE: - download_url='https://github.com/brightwind-dev/brightwind/archive/v2.3.0.tar.gz', + download_url='https://github.com/brightwind-dev/brightwind/archive/v2.4.0.tar.gz', license='MIT', author='Stephen Holleran of BrightWind Ltd', author_email='stephen@brightwindanalysis.com',