Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ Everything is in `roofcast.example.yaml`, with comments. The parts that matter m
- `power_unit`: `W` or `kW`, the unit of your sensors. With a wrong unit, `learn` logs the reason and exits with an error instead of writing a calibration.
- `inverter_ac_limit_kw`: the forecast is clipped to it.
- `curtailment`: battery SoC and grid power sensors, to drop hours where a full battery may have throttled the PV. Set both or neither. The grid sensor must be in W, whatever `power_unit` says, because `export_threshold_w` is compared with it as is.
- `curtailment.clip_limit_kw`: optional, for an array larger than its inverter. Hours where the forecast reaches 70 % of this limit or more are left out of learning, since a clipped hour shows the inverter's ceiling, not what the roof could do. Without it, those flat tops are learned as lost yield and hazy middays come out low. The hours are picked on the forecast, not on the measured value, so forecast error does not bias the level. The limit applies to the sum of all arrays. On a hybrid that sends PV above the AC rating into the battery, an hour at the AC limit is real production: leave it unset, or set it to the PV-side limit if there is one.
- `compare`: Solcast or Open-Meteo integration sensors to archive and score next to this forecast.
- `home_assistant.url`: the default is `http://homeassistant.local:8123`. A systemd service on a headless host, or a Docker container, may not be able to resolve `.local` names, so an IP address is more reliable. The add-on uses `http://supervisor/core` instead and doesn't need this.
- `models`: two of the defaults only have their own data for Europe. Outside Europe, `knmi_harmonie_arome_europe` returns no data, so it never gets a calibration: the `models` attribute shows it as `no calibration` (`failed` until the first successful `learn`) and the other models still publish. `dmi_seamless` returns the same values as ECMWF IFS there, so the ensemble gets IFS twice, next to `ecmwf_ifs025`. Drop both there.
Expand Down
3 changes: 3 additions & 0 deletions roofcast.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ publish:
# grid_power_entity: sensor.grid_power # positive = import, negative = export, in W
# soc_full_pct: 97
# export_threshold_w: -300
# clip_limit_kw: 8.0 # opt-in: hours forecast at >= 70 % of this may clip and are left out of learning.
# # On a hybrid that charges the battery above its AC rating, leave it unset
# # or use the PV-side limit.

# Optional: archive other forecasts already in HA and score them the same way (attributes *_wmape_7d/30d).
# compare:
Expand Down
2 changes: 2 additions & 0 deletions src/roofcast/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ class Config:
grid_entity: str | None = None
soc_full_pct: float = 97.0
export_threshold_w: float = -300.0
clip_limit_kw: float | None = None
solcast_entities: list[str] = field(default_factory=list)
open_meteo_entities: list[str] = field(default_factory=list)
start_efficiency: float = 0.85
Expand Down Expand Up @@ -184,6 +185,7 @@ def parse(raw: dict) -> Config:
grid_entity=cur.get('grid_power_entity'),
soc_full_pct=_num(cur, 'soc_full_pct', 'curtailment', 0, 100, default=97),
export_threshold_w=_num(cur, 'export_threshold_w', 'curtailment', default=-300),
clip_limit_kw=_num(cur, 'clip_limit_kw', 'curtailment', 0.1) if cur.get('clip_limit_kw') is not None else None,
solcast_entities=list(cmp_.get('solcast_entities') or []),
open_meteo_entities=list(cmp_.get('open_meteo_entities') or []),
start_efficiency=_num(new, 'start_efficiency', 'new_array', 0.1, 1.5, default=0.85),
Expand Down
18 changes: 13 additions & 5 deletions src/roofcast/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
MIN_QUANTILE_DAYS = 5
NEW_ARRAY_BOUNDS = (0.1, 1.5)
NWP_KEEP_DAYS = 35 # > LEVEL_DAYS + 3 days of margin used by level_day_ahead
CLIP_SHARE = 0.7 # forecast hour >= this share of curtailment.clip_limit_kw may clip


def log(*a):
Expand Down Expand Up @@ -368,6 +369,14 @@ def learn(self) -> int:
a.loc[dead, ['ok', 'okb']] = False
if dead.any():
log('learn:', int(dead.sum()), 'hours at ~0 while the forecast expected production, left out')
if ref and cfg.clip_limit_kw:
# Hours that may clip do not show what the roof could do. Chosen on the forecast, not on the measured value:
# dropping hours measured at the limit keeps those where the forecast was too high and pulls g low.
# Opt-in: on a hybrid that charges the battery above the AC rating, hours at the AC limit are real production.
near = pd.concat(ref, axis=1).mean(axis=1) * cfg.start_efficiency >= CLIP_SHARE * cfg.clip_limit_kw
a.loc[near, ['ok', 'okb']] = False
if near.any():
log('learn:', int(near.sum()), 'hours forecast near the clipping limit, left out')
ens = {}
for m in cfg.models:
for lead in (0, 1):
Expand All @@ -382,15 +391,15 @@ def learn(self) -> int:
cal['models'][f'{m}|{lead}'] = ent
if lead == 1:
kb = binkey(F)
ens[m] = sum(self.apply_cal(F, kb, ent, x) for x in cfg.arrays).clip(upper=cfg.ac_limit_kw)
ens[m] = sum(self.apply_cal(F, kb, ent, x) for x in cfg.arrays)
if not ens:
# nothing fitted tonight (API down, wrong unit, no history): never write an empty or stale-only
# calibration, and fail so systemd retries
log('learn: no model could be fitted, calibration.json not written')
return 1
# quantiles: daily actual / day-ahead ensemble ratio
if ens:
E = pd.DataFrame(ens).mean(axis=1)
E = pd.DataFrame(ens).mean(axis=1).clip(upper=cfg.ac_limit_kw) # clipped after the mean, like run()'s P50
ok = a.ok.reindex(E.index).fillna(False).astype(bool)
dd = pd.DataFrame({'f': E[ok].groupby(days[ok.values]).sum(), 'y': a.total[ok].groupby(days[ok.values]).sum()})
dd = dd[dd.f > self.min_day_kwh]
Expand Down Expand Up @@ -566,7 +575,7 @@ def run(self) -> int:
p[sel] = self.apply_cal(F, kb, ent, x)[sel]
per_array[x.name][m] = p
tot = tot + p
members[m] = tot.clip(upper=cfg.ac_limit_kw)
members[m] = tot # clipped only after k: clip(mean x k), not clip(mean(clip)) x k
status[m] = 'ok'
# archived RAW: before k and before the AC clip, so level k can be measured without feedback
c.executemany('insert or replace into nwp values(?,?,?,?,?,?,?,?,?,?,?)',
Expand All @@ -584,10 +593,9 @@ def run(self) -> int:
if not LEVEL_BOUNDS[0] <= K <= LEVEL_BOUNDS[1]:
K = 1.0 # corrupt calibration file: no correction
M = pd.DataFrame(members)
p50m = (M.mean(axis=1, skipna=True) * K).clip(upper=cfg.ac_limit_kw)
n = M.notna().sum(axis=1)
q = cal['quantiles']
base = p50m / max(q['p50'], 0.5)
base = M.mean(axis=1, skipna=True) * K / max(q['p50'], 0.5) # bands from the unclipped expectation
Q = pd.DataFrame({k: (base * q[k]).clip(upper=cfg.ac_limit_kw) for k in ('p10', 'p25', 'p50', 'p90')})
fut = Q[Q.index >= now.floor('h') - pd.Timedelta(hours=1)]
c.executemany('insert or replace into out values(?,?,?,?,?,?,?)',
Expand Down
108 changes: 108 additions & 0 deletions tests/test_clipping.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""Issue #5: an oversized array clipped at the inverter limit must not be learned as lost yield, and the level factor
must apply before the AC clip, not after it."""
import json
import sqlite3

import numpy as np
import pandas as pd
import pytest

from roofcast.config import ConfigError, parse

from conftest import FakeHA, clear_sky_weather
from test_config import mk
from test_run import NOW, eng # noqa: F401 (fixture)

LIMIT = 5.0 # kW, on the example's 10 kWp: clear days clip from mid-morning to mid-afternoon


def _history(eng, monkeypatch, noise=0.0):
"""30 clear days; the roof makes 0.9 x pvlib on each array (times hourly forecast error), measured as min(DC, AC),
split pro rata."""
cfg = eng.cfg
cfg.history_days = 30
w = clear_sky_weather(cfg, (NOW - pd.Timedelta(days=31)).floor('D'), 32 * 24)
F = eng.pv(w)
err = pd.Series(np.random.default_rng(1).normal(1.0, noise, len(F)), index=F.index).clip(lower=0.3)
F = F.mul(err, axis=0)
dc = sum(F[x.name] * 0.9 for x in cfg.arrays)
scale = (LIMIT / dc).clip(upper=1.0).fillna(1.0)
eng._ha = FakeHA({x.entities[0]: F[x.name] * 0.9 * scale * 1000 for x in cfg.arrays})
monkeypatch.setattr(eng, 'fetch_prev_runs', lambda m, lead, s, e: w)
assert (dc > LIMIT).sum() > 90 # the setup really clips


def _g(eng):
cal = json.load(open(eng.cal_path))
ent = cal['models'][f'{eng.cfg.models[0]}|1']
return ent['south']['g'], ent['west']['g']


def test_clipped_hours_learned_as_lost_yield_without_the_option(eng, monkeypatch):
_history(eng, monkeypatch)
assert eng.learn() == 0
s, w = _g(eng)
assert s < 0.88 and w < 0.88 # the flat tops pull the level down: this is what the option is for


def test_clip_limit_leaves_clipped_hours_out(eng, monkeypatch):
_history(eng, monkeypatch)
eng.cfg.clip_limit_kw = LIMIT
assert eng.actuals().ok.all() # censored in learn(), on the forecast, not on the measured hour
assert eng.learn() == 0
s, w = _g(eng)
assert s == pytest.approx(0.9, abs=0.005) and w == pytest.approx(0.9, abs=0.005)


def test_forecast_error_does_not_bias_the_level(eng, monkeypatch):
"""With hourly forecast error, dropping hours on the measured value keeps the hours where the forecast was too
high near the limit and pulls g low. Censoring on the forecast does not."""
_history(eng, monkeypatch, noise=0.15)
eng.cfg.clip_limit_kw = LIMIT
assert eng.learn() == 0
s, w = _g(eng)
assert s == pytest.approx(0.9, abs=0.02) and w == pytest.approx(0.9, abs=0.02)


def test_default_keeps_every_hour(eng, monkeypatch):
_history(eng, monkeypatch)
assert eng.cfg.clip_limit_kw is None
a = eng.actuals()
assert a.ok.all()


def test_k_applies_before_the_clip(eng):
"""k = 0.9 with every member far above the limit: P50 is the limit, not 0.9 x the limit."""
eng.cfg.ac_limit_kw = 2.0
cal = {'generated': 'x', 'n_hours': 0, 'models': {}, 'level': {'k': 0.9},
'quantiles': {'p10': 0.6, 'p25': 0.8, 'p50': 1.0, 'p90': 1.2, 'n_days': 30}}
json.dump(cal, open(eng.cal_path, 'w'))
assert eng.run() == 0
c = sqlite3.connect(eng.db_path)
t = int(pd.Timestamp('2026-10-24T11:00Z').timestamp())
raw = [r[0] for r in c.execute('select pv from nwp where target = ?', (t,))]
mean = sum(raw) / len(raw)
assert mean * 0.9 > 2.0 / 0.6 # every band is above the limit at noon
p10, p50, p90 = c.execute('select p10, p50, p90 from out where target = ?', (t,)).fetchone()
assert p50 == pytest.approx(2.0) and p10 == pytest.approx(2.0) and p90 == pytest.approx(2.0)


def test_bands_come_from_the_unclipped_expectation(eng):
"""Where the expectation straddles the limit, P10 is a fraction of the expectation, not of the limit."""
eng.cfg.ac_limit_kw = 3.0
cal = {'generated': 'x', 'n_hours': 0, 'models': {}, 'level': {'k': 1.0},
'quantiles': {'p10': 0.6, 'p25': 0.8, 'p50': 1.0, 'p90': 1.2, 'n_days': 30}}
json.dump(cal, open(eng.cal_path, 'w'))
eng.run()
c = sqlite3.connect(eng.db_path)
rows = c.execute('select o.p10, o.p50, (select avg(pv) from nwp n where n.target = o.target) from out o').fetchall()
for p10, p50, mean in rows:
assert p50 == pytest.approx(min(mean, 3.0))
assert p10 == pytest.approx(min(mean * 0.6, 3.0))


def test_clip_limit_config():
assert parse(mk(curtailment={'clip_limit_kw': 7.5})).clip_limit_kw == 7.5
assert parse(mk()).clip_limit_kw is None
with pytest.raises(ConfigError, match='clip_limit_kw'):
parse(mk(curtailment={'clip_limit_kw': 0}))
Loading