Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions clean/l/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ the frame parity, as Model M's.
- LDR writes 11/11/10-bit display codes of the plain colour. The output stage unpacks them (code RN32(1/N), FP16)
and filters them with no decode and no exposure (`recon.output_filter`).
- The output filter is Model M's (the project's own taps).
- Steps 4 to 6 (history candidates, combination, blend) run in float32 with the fused forms, one rounding per
operation, as the spec side states the order (table `"blend": "float32"`; Model M's entry keeps float64). This
matters where a large fallback colour (about 1e3 and up) is cancelled in the blend: H^ = fused((C - Hl) beta' +
Hl) rounds C - Hl to float32 first, and the float64 form lands many FP16 steps away there.

The reconstruction kernel `l_post_recon` (`l_post.co`, HDR; exists, GPU-checked by
`clean/l/runner/l_post_check.py`):
Expand Down
41 changes: 31 additions & 10 deletions clean/l/reference/recon.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ def reconstruct(x, s, t=_table.TABLE, ldr=False):
window = int(rc["window"])
linear = rc["history_combine"] == "linear"
lin_fallback = rc["fallback"] == "linear_colour"
f32 = rc.get("blend", "float64") == "float32"
r32 = _f32 if f32 else (lambda v: v)
fma = lambda a, b, c: np.asarray(_motion_hi.fma32(a, b, c), np.float64) # noqa: E731 (float32, one rounding)
enc, dec, enc_in = _transfer(ldr)

color = np.asarray(x["color"], np.float64)
Expand Down Expand Up @@ -236,7 +239,7 @@ def along(qx, qy):
hs = _post.history_sample(hist, np.where(valid, qx, 0.5), np.where(valid, qy, 0.5)) if not reset else 0.0
if linear:
with np.errstate(invalid="ignore", over="ignore"):
hs = dec(hs) / e_prev
hs = r32(r32(dec(hs)) / e_prev)
return np.where(valid[..., None], hs, fallback), nanq

if hi:
Expand All @@ -256,16 +259,33 @@ def along(qx, qy):
h2 = np.take_along_axis(Hk, second[..., None, None], -1)[..., 0]
nan_h = sel(nk, first) | ((w2 != 0) & sel(nk, second))
with np.errstate(invalid="ignore", over="ignore"):
H = np.where((w2 == 0)[..., None], w1[..., None] * h1, w1[..., None] * h1 + w2[..., None] * h2)
if f32:
# float32, one rounding per operation: Hl = f32(fused(w1 L1 + f32(w2 L2)) e) (w1 L1 alone when w2 = 0).
w1, w2 = r32(w1), r32(w2)
H = np.where((w2 == 0)[..., None], r32(w1[..., None] * h1),
fma(w1[..., None], h1, r32(w2[..., None] * h2)))
else:
H = np.where((w2 == 0)[..., None], w1[..., None] * h1, w1[..., None] * h1 + w2[..., None] * h2)

# 5. blend
C_lin = dec(cbar)
H_lin = H * e if linear else dec(H) / e_prev * e
alpha = _post.clamp(s * W * gbar, 0.0, 1.0)
bh2 = (alpha * (1.0 - bh) + bh)[..., None]
bo2 = (alpha * (1.0 - bo) + bo)[..., None]
Hh = H_lin + (C_lin - H_lin) * bh2
Oh = H_lin + (C_lin - H_lin) * bo2
if f32:
# alpha = sat(f32(f32(s W) g-bar)), beta' = fused(alpha f32(1 - beta-bar) + beta-bar),
# H^ = fused(f32(C - Hl) beta' + Hl), O^ likewise.
C_lin = r32(dec(cbar))
H_lin = r32(H * e)
alpha = _post.clamp(r32(r32(np.float32(s) * W) * gbar), 0.0, 1.0)
bh2 = fma(alpha, r32(1.0 - bh), bh)[..., None]
bo2 = fma(alpha, r32(1.0 - bo), bo)[..., None]
Hh = fma(r32(C_lin - H_lin), bh2, H_lin)
Oh = fma(r32(C_lin - H_lin), bo2, H_lin)
else:
C_lin = dec(cbar)
H_lin = H * e if linear else dec(H) / e_prev * e
alpha = _post.clamp(s * W * gbar, 0.0, 1.0)
bh2 = (alpha * (1.0 - bh) + bh)[..., None]
bo2 = (alpha * (1.0 - bo) + bo)[..., None]
Hh = H_lin + (C_lin - H_lin) * bh2
Oh = H_lin + (C_lin - H_lin) * bo2
Hh = np.where((np.isnan(Hh.sum(-1)) | nan_h)[..., None], C_lin, Hh)
Oh = np.where((np.isnan(Oh.sum(-1)) | nan_h)[..., None], C_lin, Oh)

Expand All @@ -286,5 +306,6 @@ def along(qx, qy):
disp = _post.clamp(enc(Oh), 0.0, _post.DISPLAY_MAX)
return {"history": _post.f16(enc(Hh)), "reduced": (code / 1023.0).astype(np.float32),
"display": _post.f16(disp),
"_lin": {"H_hat": Hh, "O_hat": Oh, "W": W, "first": first, "second": second, "w1": w1, "w2": w2},
"_lin": {"H_hat": Hh, "O_hat": Oh, "W": W, "first": first, "second": second, "w1": w1, "w2": w2,
"C_lin": C_lin, "H_lin": H_lin, "beta_h": bh2[..., 0], "beta_o": bo2[..., 0]},
**({"display_codes": codes.astype(np.uint16)} if codes is not None else {})}
8 changes: 5 additions & 3 deletions clean/l/reference/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,9 @@
# The reconstruction (clean/l/reference/recon.py): Model M's (spec 06) with a "window" x "window" colour gather,
# history candidates combined in linear colour ("history_combine"), and the plain linear colour as the fallback
# of an invalid or reset sample ("fallback"); "gather": "half" = the colour gather in the model's precision
# (recon.gather_half). Model M: window 3, "encoded", "encoded_colour", "float64". The output stage
# (recon.gather_half); "blend": "float32" = steps 4 to 6 (history candidates, combination, blend) in float32
# with the fused forms, one rounding per operation, as the spec side's order states (recon.reconstruct). Model M:
# window 3, "encoded", "encoded_colour", "float64", "float64". The output stage
# (D -> output filter) is Model M's unchanged. "post_variant" names Model L's kernel set (one function, both
# tilings).
"post_variant": "unfolded",
Expand All @@ -89,7 +91,7 @@
# ratio (single-precision division and comparison), else "3/2" (recon_variant). The ratio is D / render, not
# output / render: 3.0 here is about output >= 2 x render.
"recon": {"rule": "d_over_render_at_least", "ratio": 3.0, "variants": ("3/2", "3/1"), "window": 4,
"history_combine": "linear", "fallback": "linear_colour", "gather": "half"},
"history_combine": "linear", "fallback": "linear_colour", "gather": "half", "blend": "float32"},
# Window offsets (ox, oy) of an L block for frame parity 0 and 1, in pixels; the blocks run on the padded grid F
# and keep every grid position's output.
"window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))},
Expand All @@ -103,7 +105,7 @@

# Model M's reconstruction in the same terms (clean/reference/post.py reconstruct): for reductions and tests.
M_RECON = {"rule": "d_over_render_at_least", "ratio": 3.0, "variants": ("3/2", "3/1"), "window": 3,
"history_combine": "encoded", "fallback": "encoded_colour", "gather": "float64"}
"history_combine": "encoded", "fallback": "encoded_colour", "gather": "float64", "blend": "float64"}


def recon_size(output_w, output_h):
Expand Down
35 changes: 34 additions & 1 deletion clean/l/runner/l_post_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
Model M's thresholds (float32 kernel against a float64 oracle); they are to be tightened when a kernel-order model of
the reconstruction exists. The oracle's colour gather runs in the model's precision (confirmed by the spec side), so
the kernel and the oracle agree on which weights underflow: no pixel is excluded.
Special colours (the encoder's random_inputs special values: NaN, 1e3 ... 1e6, small negatives, +inf): the same
criteria; the ill-conditioned pixels (a large fallback colour cancelled in the blend) are also reported apart.
Known answers (C = (65504, 65504, 0), D = R, every target offset from a texel centre by the jitter): offset 0.1, total
weight 0 everywhere: history -1.4404296875, display 0, reduced code 0; offset 0.05, total weight > 0 but every FP16
weight 0: the output is the candidate history (here the reset fallback: the linear colour times e).
Expand Down Expand Up @@ -46,12 +48,30 @@
# (rw, rh, dw, dh, reset, kind)
CASES = [(24, 16, 54, 36, False, "random"), (31, 17, 70, 38, False, "random"), (20, 12, 60, 36, False, "random"),
(33, 19, 99, 57, True, "random"), (17, 11, 38, 25, False, "extreme_c"),
(16, 12, 16, 12, False, "offset_0.1"), (16, 12, 16, 12, True, "offset_0.05")]
(16, 12, 16, 12, False, "offset_0.1"), (16, 12, 16, 12, True, "offset_0.05"),
(24, 16, 54, 36, False, "special"), (31, 17, 70, 38, True, "special"), (33, 19, 99, 57, False, "special")]


def special_colours(rng, color):
"""The special colours of clean/reference/encoder.py random_inputs (special=True): NaN, 1e3 ... 1e6, small
negatives, +inf in red, each on 1/200 of the pixels."""
col = np.asarray(color, np.float32).copy()
n = col.shape[0] * col.shape[1]
flat = col.reshape(-1, 3)
idx = rng.choice(n, size=max(4, n // 50), replace=False)
flat[idx[0::4]] = np.nan
flat[idx[1::4]] = rng.uniform(1e3, 1e6, size=(len(idx[1::4]), 3))
flat[idx[2::4]] = -rng.uniform(0.0, 0.05, size=(len(idx[2::4]), 3))
flat[idx[3::4], 0] = np.inf
return col


def make_case(rng, rw, rh, dw, dh, reset, kind):
x = MP.random_inputs(rng, rw, rh, dw, dh, reset=reset)
x["exposure"], x["exposure_prev"] = np.float32(rng.uniform(0.5, 2.0)), np.float32(rng.uniform(0.5, 2.0))
if kind == "special":
x["color"] = special_colours(rng, x["color"])
return x
if kind != "random":
x["C"] = np.broadcast_to(np.array([65504.0, 65504.0, 0.0], np.float32), np.shape(x["C"])).copy()
if kind.startswith("offset_"):
Expand Down Expand Up @@ -131,6 +151,19 @@ def check_case(hip, mod, x, kind, s=S):
ok = m["pass"] and got["alpha_zero"] and got["reduced_top_zero"] and got["slack_untouched"]
W = ref["_lin"]["W"]
m["weight_zero_pixels"] = int((W == 0).sum())
if kind == "special":
# Ill-conditioned pixels: H^ = Hl + (C - Hl) beta' cancels a large Hl (a large fallback colour), amplifying
# every rounding of Hl by |Hl| |1 - beta'| / |H^| (here >= 2^8). Reported apart; the criteria above cover all.
lin = ref["_lin"]
with np.errstate(invalid="ignore", divide="ignore", over="ignore"):
amp = (np.abs(lin["H_lin"]) * np.abs(1.0 - lin["beta_h"][..., None])
/ np.maximum(np.abs(lin["H_hat"]), 1e-30)).max(-1)
ill = np.nan_to_num(amp, nan=0.0) >= 256.0
st = C32.f16_steps(np.where(np.isfinite(got["history"]), got["history"], 0), ref["history"])
m["ill_conditioned"] = {"pixels": int(ill.sum()), "max_steps": int(st[ill].max(initial=0)),
"over_1": int((st[ill] > 1).sum())}
m["well_conditioned"] = {"max_steps": int(st[~ill].max(initial=0)), "over_1": int((st[~ill] > 1).sum()),
"values": int(st[~ill].size)}
if kind == "offset_0.1":
known = ((got["history"] == np.float32(-1.4404296875)).all() and (got["display"] == 0).all()
and (got["reduced_codes"] == 0).all() and m["weight_zero_pixels"] == W.size)
Expand Down
63 changes: 52 additions & 11 deletions clean/l/tests/test_l_recon.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,31 @@
from clean.l.reference import oracle as L
from clean.l.reference import recon as R
from clean.l.reference import table as T
from clean.reference import motion_hi as MH
from clean.reference import post as MP


def _comb32(w1, w2, h1, h2, e):
"""Model L's combination in float32 (the spec side's order): f32(fused(w1 h1 + f32(w2 h2)) e), w1 h1 alone when
w2 = 0, w1 and w2 rounded to float32 first."""
w1, w2 = R._f32(w1)[..., None], R._f32(w2)[..., None]
c = np.where(w2 == 0, R._f32(w1 * h1), np.asarray(MH.fma32(w1, h1, R._f32(w2 * h2)), np.float64))
return R._f32(c * e)


def _same(a, b):
a, b = np.asarray(a), np.asarray(b)
return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes()


def _f16_steps(a, b):
"""Distance in FP16 steps (ordered bit patterns) of FP16-exact arrays."""
ia, ib = (np.asarray(v, np.float32).astype(np.float16).view(np.int16).astype(np.int64) for v in (a, b))
ia = np.where(ia < 0, -(ia & 0x7FFF), ia)
ib = np.where(ib < 0, -(ib & 0x7FFF), ib)
return np.abs(ia - ib)


def _inputs(rng, rw=23, rh=15, reset=False):
x = MP.random_inputs(rng, rw, rh, reset=reset)
x["exposure"], x["exposure_prev"] = np.float32(1.7), np.float32(0.6)
Expand Down Expand Up @@ -97,8 +114,7 @@ def test_reset_uses_the_linear_colour(self):
dw, dh = x["recon_size"]
px, py = R.target_positions_f32(rw, rh, dw, dh, *np.asarray(x["jitter"], np.float64))
fy, fx = MP.point_index(px, py, rw, rh)
w = lin["w1"] + np.where(lin["w2"] == 0, 0.0, lin["w2"])
H_lin = w[..., None] * color[fy, fx] * float(x["exposure"])
H_lin = _comb32(lin["w1"], lin["w2"], color[fy, fx], color[fy, fx], float(x["exposure"]))
self.assertTrue(np.isfinite(lin["H_hat"]).all())
m = R.reconstruct(x, 2.0, T.table(recon=dict(T.M_RECON)))
self.assertFalse(np.allclose(m["_lin"]["H_hat"], lin["H_hat"]))
Expand All @@ -108,7 +124,8 @@ def test_reset_uses_the_linear_colour(self):
# s = 0 gives alpha = 0: H^ = H_lin + (C_lin - H_lin) beta_h, linear in H_lin; with beta_h = 0 it is H_lin.
x0 = dict(x, A=np.zeros_like(np.asarray(x["A"]))) # beta_h = beta_o = 0 (and g = 0)
z = R.reconstruct(x0, 0.0)
np.testing.assert_allclose(z["_lin"]["H_hat"], H_lin, rtol=1e-12)
H_lin = _comb32(z["_lin"]["w1"], z["_lin"]["w2"], color[fy, fx], color[fy, fx], float(x["exposure"]))
self.assertTrue(np.array_equal(z["_lin"]["H_hat"], H_lin)) # beta' = 0 exactly: H^ = Hl

def test_linear_combination(self):
"""A valid sample is decoded and divided by e_prev before the two candidates are combined; the result is
Expand All @@ -133,14 +150,19 @@ def test_linear_combination(self):
for mk in m:
qx, qy = X + mk[..., 0], Y + mk[..., 1]
valid = (qx >= 0) & (qx <= dw) & (qy >= 0) & (qy <= dh)
hs = MP.dec(MP.history_sample(hist, np.where(valid, qx, 0.5), np.where(valid, qy, 0.5))) / ep
hs = R._f32(R._f32(MP.dec(MP.history_sample(hist, np.where(valid, qx, 0.5), np.where(valid, qy, 0.5))))
/ ep)
cands.append(np.where(valid[..., None], hs, color[fy, fx]))
Hk = np.stack(cands, -1)
h1 = np.take_along_axis(Hk, lin["first"][..., None, None], -1)[..., 0]
h2 = np.take_along_axis(Hk, lin["second"][..., None, None], -1)[..., 0]
w1, w2 = lin["w1"][..., None], lin["w2"][..., None]
H_lin = np.where(w2 == 0, w1 * h1, w1 * h1 + w2 * h2) * e
np.testing.assert_allclose(lin["H_hat"], H_lin, rtol=1e-12)
self.assertTrue(np.array_equal(lin["H_hat"], _comb32(lin["w1"], lin["w2"], h1, h2, e)))
# The same with the blend in float64 (Model M's precision): the float64 combination.
lin64 = R.reconstruct(x0, 0.0, T.table(recon=dict(T.TABLE["recon"], blend="float64")))["_lin"]
w1, w2 = lin64["w1"][..., None], lin64["w2"][..., None]
h1d = np.take_along_axis(Hk, lin64["first"][..., None, None], -1)[..., 0]
h2d = np.take_along_axis(Hk, lin64["second"][..., None, None], -1)[..., 0]
np.testing.assert_allclose(lin64["H_hat"], np.where(w2 == 0, w1 * h1d, w1 * h1d + w2 * h2d) * e, rtol=1e-6)

def test_ldr_reads_no_exposure(self):
rng = np.random.default_rng(7)
Expand Down Expand Up @@ -231,11 +253,11 @@ def test_offset_005_is_the_candidate_history(self):
self.assertTrue((W > 0).all() and np.isfinite(W).all())
e, ep = 1.5, 0.75
if reset:
Hl = np.asarray(x["color"], np.float64) * e # D = R, floor p = the pixel itself
Hl = R._f32(np.asarray(x["color"], np.float64) * e) # D = R, floor p = the pixel itself; w1 = 1
else:
Hl = np.full(out["history"].shape, float(MP.dec(0.75)) / ep * e)
np.testing.assert_allclose(out["_lin"]["H_hat"], Hl, rtol=1e-12)
np.testing.assert_allclose(out["_lin"]["O_hat"], Hl, rtol=1e-12)
Hl = np.full(out["history"].shape, R._f32(R._f32(R._f32(MP.dec(0.75)) / ep) * e))
self.assertTrue(np.array_equal(out["_lin"]["H_hat"], Hl)) # alpha = 0, beta-bar = 0: H^ = Hl
self.assertTrue(np.array_equal(out["_lin"]["O_hat"], Hl))

def test_offset_01_is_zero_weight(self):
"""Offset 0.1 (beyond 0.0516): W = 0 everywhere, the confirmed zero-weight values."""
Expand All @@ -246,6 +268,25 @@ def test_offset_01_is_zero_weight(self):
self.assertTrue((out["display"] == 0).all())
self.assertTrue((out["reduced"] == 0).all())

def test_blend_in_float32_with_large_colours(self):
"""Steps 4 to 6 in float32 with the fused forms (the spec side's order): H^ = fused(f32(C - Hl) beta' + Hl)
exactly, also where the fallback colour is large (1e3 ... 1e6, reset: every sample is the fallback). There
H^ comes from cancelling large terms, so the float64 form lands several FP16 steps away: the case of the
frame check's special colours."""
rng = np.random.default_rng(14)
x = _inputs(rng, reset=True)
col = np.asarray(x["color"], np.float32).copy()
big = rng.random(col.shape[:2]) < 0.1
col[big] = rng.uniform(1e3, 1e6, (int(big.sum()), 3))
x["color"] = col
lin = R.reconstruct(x, 2.9796233)["_lin"]
C, Hl, bh, bo = lin["C_lin"], lin["H_lin"], lin["beta_h"][..., None], lin["beta_o"][..., None]
self.assertTrue(np.array_equal(lin["H_hat"], np.asarray(MH.fma32(R._f32(C - Hl), bh, Hl), np.float64)))
self.assertTrue(np.array_equal(lin["O_hat"], np.asarray(MH.fma32(R._f32(C - Hl), bo, Hl), np.float64)))
out64 = R.reconstruct(x, 2.9796233, T.table(recon=dict(T.TABLE["recon"], blend="float64")))
st = _f16_steps(R.reconstruct(x, 2.9796233)["history"], out64["history"])
self.assertGreater(int(st.max()), 1)

def test_history_of_another_size_needs_reset(self):
rng = np.random.default_rng(11)
x = _inputs(rng)
Expand Down
Loading