Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ Everything is in `roofcast.example.yaml`, with comments. The parts that matter m
- `power_unit`: `W` or `kW`, the unit of your sensors. With a wrong unit, `learn` logs the reason and exits with an error instead of writing a calibration.
- `inverter_ac_limit_kw`: the forecast is clipped to it.
- `curtailment`: battery SoC and grid power sensors, to drop hours where a full battery may have throttled the PV. Set both or neither. The grid sensor must be in W, whatever `power_unit` says, because `export_threshold_w` is compared with it as is.
- `curtailment.clip_limit_kw`: optional, for an array larger than its inverter. Hours where the forecast reaches 70 % of this limit or more are left out of learning, since a clipped hour shows the inverter's ceiling, not what the roof could do. Without it, those flat tops are learned as lost yield and hazy middays come out low. The hours are picked on the forecast, not on the measured value, so forecast error does not bias the level. The limit applies to the sum of all arrays. On a hybrid that sends PV above the AC rating into the battery, an hour at the AC limit is real production: leave it unset, or set it to the PV-side limit if there is one.
- `curtailment.clip_limit_kw`: optional, for an array larger than its inverter. Hours where the day-ahead forecast (raw pvlib times `new_array.start_efficiency`) reaches 70 % of this limit or more are left out of learning, and out of `band_coverage`, since a clipped hour shows the inverter's ceiling, not what the roof could do. Without it, those flat tops are learned as lost yield and hazy middays come out low. The hours are picked on the forecast because picking them on the measured value pulls the level down: on a roof that never clips, dropping the top 20 % of hours gave 0.89 against a true 0.90 when picked on the forecast, and 0.79 when picked on the measured value. Some bias is left when the forecast is very noisy, and the filter only catches clipping while the real efficiency stays under about `start_efficiency` / 0.7. The limit applies to the sum of all arrays. On a hybrid that sends PV above the AC rating into the battery, an hour at the AC limit is real production: leave it unset, or set it to the PV-side limit if there is one.
- `compare`: Solcast or Open-Meteo integration sensors to archive and score next to this forecast.
- `home_assistant.url`: the default is `http://homeassistant.local:8123`. A systemd service on a headless host, or a Docker container, may not be able to resolve `.local` names, so an IP address is more reliable. The add-on uses `http://supervisor/core` instead and doesn't need this.
- `models`: two of the defaults only have their own data for Europe. Outside Europe, `knmi_harmonie_arome_europe` returns no data, so it never gets a calibration: the `models` attribute shows it as `no calibration` (`failed` until the first successful `learn`) and the other models still publish. `dmi_seamless` returns the same values as ECMWF IFS there, so the ensemble gets IFS twice, next to `ecmwf_ifs025`. Drop both there.
Expand Down
18 changes: 12 additions & 6 deletions src/roofcast/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
# Used until enough clear days exist to learn the spread (cold start). P50 stays the ensemble mean.
DEFAULT_QUANTILES = {'p10': 0.55, 'p25': 0.8, 'p50': 1.0, 'p90': 1.25}
MIN_QUANTILE_DAYS = 5
COVERAGE_DAYS, MIN_COVERAGE_DAYS = 60, 7
BAND_EPS_KWH = 0.01 # a band narrower than this (clipped at the AC limit) is not scored # band coverage: window, and fewer scored days than this = no percentages
COVERAGE_DAYS, MIN_COVERAGE_DAYS = 60, 7 # band coverage: window, and fewer scored days than this = no percentages
BAND_EPS_KWH = 0.01 # a band narrower than this (clipped at the AC limit) is not scored
NEW_ARRAY_BOUNDS = (0.1, 1.5)
NWP_KEEP_DAYS = 35 # > LEVEL_DAYS + 3 days of margin used by level_day_ahead
CLIP_SHARE = 0.7 # forecast hour >= this share of curtailment.clip_limit_kw may clip
Expand Down Expand Up @@ -354,7 +354,7 @@ def learn(self) -> int:
if end not in f:
os.remove(os.path.join(self.cache, f))
prev = self.load_cal() or {}
cal = {'generated': self.now().isoformat(), 'n_hours': int(a.ok.sum()), 'models': {}}
cal = {'generated': self.now().isoformat(), 'models': {}}
days = np.array(a.index.tz_convert(self.tz).date)
# The outage filter in actuals() reads the live archive, which a fresh install does not have yet. Apply the same
# rule with the day-ahead previous runs, or hours at ~0 before the panels were installed (or with the inverter
Expand All @@ -363,22 +363,28 @@ def learn(self) -> int:
for m in cfg.models:
try:
F = self.pv(self.fetch_prev_runs(m, 1, start, end).reindex(a.index).interpolate(limit=2))
ref.append(sum(F[x.name] for x in cfg.arrays))
ref.append(sum(F[x.name] for x in cfg.arrays).where(F.has_nwp)) # a model with no data is not a 0 forecast
except Exception:
pass # the model loop below logs the failure
ref = [r for r in ref if r.notna().any()] # empty frames count as no forecast
if ref:
dead = (a.total < 0.02) & (pd.concat(ref, axis=1).mean(axis=1) > self.dead_hour_kwh)
a.loc[dead, ['ok', 'okb']] = False
if dead.any():
log('learn:', int(dead.sum()), 'hours at ~0 while the forecast expected production, left out')
if ref and cfg.clip_limit_kw:
# Hours that may clip do not show what the roof could do. Chosen on the forecast, not on the measured value:
# dropping hours measured at the limit keeps those where the forecast was too high and pulls g low.
# Hours that may clip do not show what the roof could do. Chosen on the forecast (raw pvlib x start_efficiency):
# g is fitted as actual given forecast, so selecting on the forecast leaves it unbiased as long as the forecast
# is roughly calibrated, while selecting on the measured value pulls it low (on a roof that never clips:
# 0.89 vs 0.90 dropping the top 20 % on the forecast, 0.79 dropping them on the measured value).
# Opt-in: on a hybrid that charges the battery above the AC rating, hours at the AC limit are real production.
near = pd.concat(ref, axis=1).mean(axis=1) * cfg.start_efficiency >= CLIP_SHARE * cfg.clip_limit_kw
a.loc[near, ['ok', 'okb']] = False
if near.any():
log('learn:', int(near.sum()), 'hours forecast near the clipping limit, left out')
elif cfg.clip_limit_kw:
log('learn: no day-ahead forecast for the clipping filter, clipped hours stay in learning tonight')
cal['n_hours'] = int(a.ok.sum())
ens = {}
for m in cfg.models:
for lead in (0, 1):
Expand Down
16 changes: 15 additions & 1 deletion tests/test_clipping.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def test_clip_limit_leaves_clipped_hours_out(eng, monkeypatch):
assert s == pytest.approx(0.9, abs=0.005) and w == pytest.approx(0.9, abs=0.005)


def test_forecast_error_does_not_bias_the_level(eng, monkeypatch):
def test_measured_noise_does_not_bias_the_level(eng, monkeypatch):
"""With hourly forecast error, dropping hours on the measured value keeps the hours where the forecast was too
high near the limit and pulls g low. Censoring on the forecast does not."""
_history(eng, monkeypatch, noise=0.15)
Expand Down Expand Up @@ -106,3 +106,17 @@ def test_clip_limit_config():
assert parse(mk()).clip_limit_kw is None
with pytest.raises(ConfigError, match='clip_limit_kw'):
parse(mk(curtailment={'clip_limit_kw': 0}))


def test_a_model_with_no_data_does_not_hide_clipping(eng, monkeypatch):
"""A model that returns nothing (outside its area) must not pull the forecast used to find clipped hours to 0."""
_history(eng, monkeypatch)
eng.cfg.clip_limit_kw = LIMIT
assert len(eng.cfg.models) >= 2
w = eng.fetch_prev_runs(None, 1, None, None)
empty = w.copy()
empty[:] = float('nan')
monkeypatch.setattr(eng, 'fetch_prev_runs', lambda m, lead, s, e: empty if m == eng.cfg.models[-1] else w)
assert eng.learn() == 0
s, w_ = _g(eng)
assert s == pytest.approx(0.9, abs=0.005) and w_ == pytest.approx(0.9, abs=0.005)
Loading