diff --git a/clean/l/README.md b/clean/l/README.md index 34ad4e58..9a7eb530 100644 --- a/clean/l/README.md +++ b/clean/l/README.md @@ -6,7 +6,7 @@ launcher yet. | Path | Content | |---|---| | `reference/table.py` | Model L's structure table: the encoder's and the decoder's op lists (pixel-level linears and width-32 block, the 2 x 2 gather projection; the decoder projection, its width-32 block and the value positions), window offsets, block constants; parameter shapes per converter-map stage; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | -| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`, parameters grouped by the converter map's stages; the decoder's coefficient functions are Model M's. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | +| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`, parameters grouped by the converter map's stages; the decoder's coefficient functions are Model M's. Both width-32 blocks run in the FP16 form (`_block_tokens_q7`, any width 32 / 64 / 96 / 128: FP16 accumulators per 32-wide FP8 op, polynomial-and-bit-shift attention weights, the FP16 GELU polynomial, the FP16 norm tree with sqrt(C) in the gain, residuals as accumulator starts; table consts `form`); the decoder block's input is RN16(projection + skip), the skip being the encoder block's FP8 output read with its own fine mirror (table `skip_map`, swin op `skip`). The arithmetic inside one FP8 matrix op is the table's `mm8` model: `exact` (the reference) or `trunc13`; records name it. The ten shared stages still run Model M's oracle (table PROVISIONAL `S8`). The C32 kernels do not yet compute the FP16 form (`l_q7_negative_control.py` shows it). The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | | `runner/l_pack.py` | weight packing of Model L's own launches: the encoder's launch list derived from the table (`encoder_launches`) and one blob per launch (`pack_model_l`), each in a format Model M already uses (merge / expand section, Swin block section, enc0 and dec0 blobs); decoders for tests. The ten shared stages use `clean/runner/packing.py` unchanged | | `kernels/l_swin.hip` | the C = 32 Swin block kernels (`l_swin_block_c32[_nc]`) from Model M's block template (`clean/kernels/swin.hip`, included for its templates only), same blob format and launch geometry as Model M's block kernels | | `runner/l_c32_check.py` | GPU check (synthetic): the C = 32 kernel against the oracle, `_nc` against the clamping kernel, and this file's C = 64 build of the template against Model M's `swin_block_c64` | diff --git a/clean/l/records/l_q7_oracle_negative_control.json b/clean/l/records/l_q7_oracle_negative_control.json new file mode 100644 index 00000000..0629073b --- /dev/null +++ b/clean/l/records/l_q7_oracle_negative_control.json @@ -0,0 +1,772 @@ +{ + "what": "negative control of Model L's oracle block form: one recorded run checked against the Model M-form oracle and the FP16-form oracle in one process", + "command": "python clean/l/runner/l_q7_negative_control.py --build --mm8 exact --json ", + "runtime_commit": "4ef40d2223bc6be232d67793025daaaf73bc1a0a", + "identity_verified": { + "git_commit": "4ef40d2223bc6be232d67793025daaaf73bc1a0a", + "git_tracked_changes": false, + "runtime_library_sha256": "594fbff57ae3f4020393e71c473e6340abd21ce979711199d811e1da2c14192c", + "params_pkl_sha256": "26874860dec760e466379f76450228a7b6d00c3018fa07180c212bba59f661f9", + "source_json_sha256": "5056738fd1287935bd924949c7a9697bf3c7c6fafe01fc6c1b6f559a2cbb8ecd" + }, + "recorded_run": { + "file": "lframe_L_stages.pkl", + "sha256": "a3d098bc39397b8131dc2d95a3ebd190353fc44306f4dc52cebe218d13125099", + "frames": 11, + "model": "L", + "kernels": { + "swin": "swin.co", + "enc0": "encoder.co", + "dec0": "decoder.co", + "exposure": "base", + "post": "post_recon", + "filter": "post_filter_c1_s4", + "post_last": "l_post_recon", + "filter_last": "post_filter_c1_s4", + "history_bytes": "8", + "history_format": "h8", + "post_bands": "0", + "post_tf": "68", + "out": "f32", + "post_wave": "64", + "motion_outres": "off", + "motion_last": "render", + "recon": "spec", + "graph": "0", + "stream_priority": "default", + "swin_nc": "enc1,enc2,b0,b1,b2,b3,b4,b5,dec2,dec1", + "swin_stages": "none", + "swin_fused": "dec2,dec1", + "swin_bneck": "none", + "swin_resident": "0", + "d1d0_p12": "0", + "dec_rdiv": "1", + "dec_bdiv": "1", + "dec1_dec0": "separate", + "swin_mid": "none", + "post_pack": "none", + "pre_exposure": "scale", + "output_filter": "analytic", + "output_taps": "none", + "model_map": "same", + "model": "L", + "l_kernels": "l_enc_values.co,l_ops.co,l_swin.co,l_post.co", + "l_post": "l_post_recon" + }, + "frame_sha256": { + "inputs": "bb13c82b0da0a5f306aae21a02507c5e2458a0e0f91b7989a48f65872c990dd0", + "outputs_raw": "843c86d99a05bf6e9e07b408efd2609b7a1ae81633988c3e03bdadda88a67359" + } + }, + "model_source": { + "kind": "converter", + "model_sha256": "d3c040191ec7ab0164efc03ee76413277e7fb54e6e6d2b168f2c0202aa1ad806", + "map_sha256": "58e6d0e7cd6926817cfea28a0b4a65c056e9bfa5ca62aa05b968b62590c9e857" + }, + "fp8_op_model": "exact", + "dec0_rule": "A, B and latent codes and C values each at most 0.5 % differing (l_frame_check dec0)", + "forms": { + "model_m_form": { + "fp8_op_model": "exact", + "dec0_frames_passed": 11, + "enc0_frames_passed": 11, + "alignment_frames_ok": 11, + "frames": [ + { + "frame": 0, + "render": [ + 96, + 64 + ], + "parity": 0, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 2 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 1, + "render": [ + 96, + 64 + ], + "parity": 1, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 2, + "codes_B": 2, + "codes_L": 2, + "C": 2 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 2, + "render": [ + 96, + 64 + ], + "parity": 0, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 4 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 3, + "render": [ + 96, + 64 + ], + "parity": 1, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 1, + "codes_L": 4, + "C": 4 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 4, + "render": [ + 96, + 64 + ], + "parity": 0, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 0 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 5, + "render": [ + 80, + 48 + ], + "parity": 1, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 1, + "codes_B": 2, + "codes_L": 4, + "C": 3 + }, + "dec0_values": { + "codes_A": 11520, + "codes_B": 11520, + "codes_L": 15360, + "C": 11520 + } + }, + { + "frame": 6, + "render": [ + 80, + 48 + ], + "parity": 0, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 0 + }, + "dec0_values": { + "codes_A": 11520, + "codes_B": 11520, + "codes_L": 15360, + "C": 11520 + } + }, + { + "frame": 7, + "render": [ + 80, + 48 + ], + "parity": 1, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 4 + }, + "dec0_values": { + "codes_A": 11520, + "codes_B": 11520, + "codes_L": 15360, + "C": 11520 + } + }, + { + "frame": 8, + "render": [ + 77, + 45 + ], + "parity": 0, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 3, + "codes_B": 3, + "codes_L": 2, + "C": 6 + }, + "dec0_values": { + "codes_A": 10395, + "codes_B": 10395, + "codes_L": 13860, + "C": 10395 + } + }, + { + "frame": 9, + "render": [ + 77, + 45 + ], + "parity": 1, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 1 + }, + "dec0_values": { + "codes_A": 10395, + "codes_B": 10395, + "codes_L": 13860, + "C": 10395 + } + }, + { + "frame": 10, + "render": [ + 77, + 45 + ], + "parity": 0, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": true, + "dec0_pass": true, + "dec0_differ": { + "codes_A": 0, + "codes_B": 0, + "codes_L": 0, + "C": 2 + }, + "dec0_values": { + "codes_A": 10395, + "codes_B": 10395, + "codes_L": 13860, + "C": 10395 + } + } + ] + }, + "fp16_form": { + "fp8_op_model": "exact", + "dec0_frames_passed": 0, + "enc0_frames_passed": 0, + "alignment_frames_ok": 11, + "frames": [ + { + "frame": 0, + "render": [ + 96, + 64 + ], + "parity": 0, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 17903, + "codes_B": 17565, + "codes_L": 24384, + "C": 18390 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 1, + "render": [ + 96, + 64 + ], + "parity": 1, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 18009, + "codes_B": 13910, + "codes_L": 24252, + "C": 18405 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 2, + "render": [ + 96, + 64 + ], + "parity": 0, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 17903, + "codes_B": 17761, + "codes_L": 24386, + "C": 18381 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 3, + "render": [ + 96, + 64 + ], + "parity": 1, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 18089, + "codes_B": 13930, + "codes_L": 24238, + "C": 18408 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 4, + "render": [ + 96, + 64 + ], + "parity": 0, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 18021, + "codes_B": 13961, + "codes_L": 24316, + "C": 18394 + }, + "dec0_values": { + "codes_A": 18432, + "codes_B": 18432, + "codes_L": 24576, + "C": 18432 + } + }, + { + "frame": 5, + "render": [ + 80, + 48 + ], + "parity": 1, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 11172, + "codes_B": 10986, + "codes_L": 15249, + "C": 11490 + }, + "dec0_values": { + "codes_A": 11520, + "codes_B": 11520, + "codes_L": 15360, + "C": 11520 + } + }, + { + "frame": 6, + "render": [ + 80, + 48 + ], + "parity": 0, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 11309, + "codes_B": 8586, + "codes_L": 15191, + "C": 11500 + }, + "dec0_values": { + "codes_A": 11520, + "codes_B": 11520, + "codes_L": 15360, + "C": 11520 + } + }, + { + "frame": 7, + "render": [ + 80, + 48 + ], + "parity": 1, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 11252, + "codes_B": 9083, + "codes_L": 15201, + "C": 11498 + }, + "dec0_values": { + "codes_A": 11520, + "codes_B": 11520, + "codes_L": 15360, + "C": 11520 + } + }, + { + "frame": 8, + "render": [ + 77, + 45 + ], + "parity": 0, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 10129, + "codes_B": 9957, + "codes_L": 13751, + "C": 10367 + }, + "dec0_values": { + "codes_A": 10395, + "codes_B": 10395, + "codes_L": 13860, + "C": 10395 + } + }, + { + "frame": 9, + "render": [ + 77, + 45 + ], + "parity": 1, + "reset": true, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 10078, + "codes_B": 9930, + "codes_L": 13753, + "C": 10368 + }, + "dec0_values": { + "codes_A": 10395, + "codes_B": 10395, + "codes_L": 13860, + "C": 10395 + } + }, + { + "frame": 10, + "render": [ + 77, + 45 + ], + "parity": 0, + "reset": false, + "alignment": { + "n_parity_reset": true, + "jitter_prev": true, + "offsets": true, + "latent_handed_over": true, + "motion_history_handed_over": true, + "history_handed_over": true, + "pass": true + }, + "enc0_features_pass": false, + "dec0_pass": false, + "dec0_differ": { + "codes_A": 10177, + "codes_B": 7343, + "codes_L": 13697, + "C": 10377 + }, + "dec0_values": { + "codes_A": 10395, + "codes_B": 10395, + "codes_L": 13860, + "C": 10395 + } + } + ] + } + } +} diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py index 8c6c4cf8..f0d609f4 100644 --- a/clean/l/reference/oracle.py +++ b/clean/l/reference/oracle.py @@ -53,7 +53,10 @@ def soft_cap(S, cap): def _block_tokens(xw, p, heads, k): - """One block on gathered windows xw [..., T, C] (T = window^2), constants k. Same step order as Model M's.""" + """One block on gathered windows xw [..., T, C] (T = window^2), constants k. k["form"] "m" (default): Model M's + step order in float64 with Model M's roundings; "q7": the FP16 form of _block_tokens_q7.""" + if k.get("form", "m") == "q7": + return _block_tokens_q7(xw, p, heads, k) C = xw.shape[-1] T = xw.shape[-2] lead = xw.shape[:-2] @@ -73,6 +76,229 @@ def _block_tokens(xw, p, heads, k): return x1 + h @ p["W2"].T + p["b2"] +# ---- the block in its FP16 form ("q7") ------------------------------------------------------------------------------ +# +# Every FP16 step rounds after each single operation (f16), FMA16 rounds once. An FP8 matrix op takes up to 32 FP8 x FP8 +# products and an FP16 start value and gives an FP16 result; a longer sum is a chain of such ops over 32-wide chunks in +# ascending order, each op's result the next one's start. What happens inside one op is not settled; two models +# (MM8_MODELS, chosen by the table's "mm8" entry, named in every record): +# - "exact" (the default): the exact sum of the products and the start value, rounded once to FP16 (exact in float64: +# the terms are multiples of 2^-24 below 2^23); +# - "trunc13": two blocks of 16 products, the start value in block 1; in each block every term is truncated toward +# zero to 13 fractional bits below the largest term's exponent, the truncated terms summed exactly and rounded to +# FP16; block 1's result is block 2's start value. +# The attention product P V (FP16 operands, 16 terms) is summed in float64 and rounded once (its accumulation precision +# does not move an FP16 result). + +MM8_MODELS = ("exact", "trunc13") + + +def _trunc13_block(start, prods): + """start [...], prods [..., k] (exact values): every term truncated toward zero to a multiple of 2^(E - 13), E the + largest term's exponent, then the exact sum rounded to FP16. Non-finite terms: the plain sum, rounded.""" + terms = np.concatenate([np.asarray(start, np.float64)[..., None], prods], axis=-1) + a = np.abs(terms) + with np.errstate(divide="ignore", invalid="ignore", over="ignore"): + _, e = np.frexp(np.where(np.isfinite(a), a, 0.0)) + e = np.where(a > 0, e - 1, np.iinfo(np.int32).min // 2) + emax = e.max(axis=-1, keepdims=True) + quantum = np.ldexp(1.0, np.maximum(emax - 13, -1074)) + tt = np.trunc(terms / quantum) * quantum + out = f16(tt.sum(axis=-1)) + bad = ~np.all(np.isfinite(terms), axis=-1) + if bad.any(): + out = np.where(bad, f16(terms.sum(axis=-1)), out) + return out + + +def _op8(start, a8, W8, model): + """One FP8 matrix op: start [..., n] (FP16), a8 [..., k] (k <= 32), W8 [n, k] -> FP16 [..., n].""" + start = np.broadcast_to(np.asarray(start, np.float64), a8.shape[:-1] + (W8.shape[0],)) + with np.errstate(over="ignore", invalid="ignore"): + if model == "exact": + return f16(start + a8 @ W8.T) + if model != "trunc13": + raise ValueError(f"unknown FP8 op model {model!r}") + out = np.empty(start.shape) + for r0 in range(0, W8.shape[0], 8): # rows in groups (the products of 8 rows at a time) + w = W8[r0:r0 + 8] + prods = a8[..., None, :] * w # [..., rows, k] + acc = start[..., r0:r0 + 8] + for b0 in (0, 16): + acc = _trunc13_block(acc, prods[..., b0:b0 + 16]) + out[..., r0:r0 + 8] = acc + return out + + +def mm8(start, a8, W8, model="exact"): + """The chain of FP8 matrix ops over the 32-wide chunks of K, ascending: a8 [..., K], W8 [n, K], start [..., n] or + [n] (FP16) -> FP16 [..., n].""" + a8 = np.asarray(a8, np.float64) + W8 = np.asarray(W8, np.float64) + acc = start + for c0 in range(0, W8.shape[1], 32): + acc = _op8(acc, a8[..., c0:c0 + 32], W8[:, c0:c0 + 32], model) + return acc + +def _h(bits): + """The FP16 constant with this bit pattern, as float64.""" + return float(np.array([bits], np.uint16).view(np.float16)[0]) + + +Q7_SCORE_SCALE = _h(0x2466) +Q7_SCORE_CLAMP = _h(0x3873) +Q7_POLY_T = _h(0x3B6B) +Q7_POLY_P = _h(0x3D80) +Q7_NORM_EPS = 2.0 ** -13 # 32 x the per-channel epsilon (the norm sums squares, it does not average) +Q7_GELU_A = _h(0x3698) +Q7_GELU_B = _h(0x2D30) + + +def _rsqrt32(v): + """The single-precision reciprocal square root of float32(v) (its last bit does not move the FP16 result).""" + with np.errstate(divide="ignore", invalid="ignore"): + s = np.asarray(v, np.float64).astype(np.float32) + return (np.float32(1.0) / np.sqrt(s)).astype(np.float64) + + +def _rcp32(v): + with np.errstate(divide="ignore", invalid="ignore"): + return (np.float32(1.0) / np.asarray(v, np.float64).astype(np.float32)).astype(np.float64) + + +def gamma_q7(gamma, C=None): + """g' = RN16(float32(gamma) x sqrtf(C)), the norm's gain with the sqrt(C) of its sum of squares folded in (C = the + block width, default len(gamma)).""" + gamma = np.asarray(gamma, np.float64) + C = gamma.shape[-1] if C is None else C + g = gamma.astype(np.float32) * np.sqrt(np.float32(C)) + return f16(g.astype(np.float32)) + + +NORM_WIDTHS = (32, 64, 96, 128) + + +def _group(ps): + """The fixed sum tree of 1 to 4 FP16 partial sums: p0; p0 + p1; (p0 + p1) + p2; (p0 + p1) + (p2 + p3).""" + if len(ps) == 1: + return ps[0] + if len(ps) == 2: + return f16(ps[0] + ps[1]) + if len(ps) == 3: + return f16(f16(ps[0] + ps[1]) + ps[2]) + return f16(f16(ps[0] + ps[1]) + f16(ps[2] + ps[3])) + + +def norm_q7(x, gp): + """RMS norm over the last axis (C = 32, 64, 96 or 128 channels) of FP16 values x with gain gp (gamma_q7): + FP16 squares q; channel tiles of 8, lane t = 0 ... 3 holding channels 8 j + 2 t (even half) and 8 j + 2 t + 1 (odd + half) of every tile j; per lane and half the pairs (tile j, tile j + 2) of the even tiles (j = 0, 4, 8, 12) form the + left group and those of the odd tiles (j = 1, 5, 9, 13) the right one, each summed by _group, left + right; across + lanes ((0 + 1) + (2 + 3)) per half, even + odd, + 2^-13; r = RN16 of the single-precision reciprocal square root; + then x RN16(r gp). Every + is one FP16 rounding.""" + x = np.asarray(x, np.float64) + C = x.shape[-1] + if C not in NORM_WIDTHS: + raise ValueError(f"norm_q7: {C} channels (32, 64, 96 or 128)") + tiles = C // 8 + with np.errstate(over="ignore", invalid="ignore"): + q = f16(x * x) + + def lane(t, odd): + Q = [q[..., 8 * j + 2 * t + odd] for j in range(tiles)] + left = _group([f16(Q[j] + Q[j + 2]) for j in range(0, tiles, 4)]) + right = _group([f16(Q[j] + Q[j + 2]) for j in range(1, tiles, 4)]) + return f16(left + right) + + def half(odd): + e = [lane(t, odd) for t in range(4)] + return f16(f16(e[0] + e[1]) + f16(e[2] + e[3])) + + s = f16(f16(half(0) + half(1)) + Q7_NORM_EPS) + r = f16(_rsqrt32(s)) + return f16(x * f16(r[..., None] * gp)) + + +def softmax_q7(s): + """Attention weights over the last axis (16 keys) from FP16 scores s: u = RN16(s c) clamped to +-c', the FP16 + polynomial p = FMA16(u, FMA16(u, -u, t0), p0) in [1, 2), the weight e = the FP16 number with bits (mantissa bits + of p) << 5, the FP16 row sum in a fixed tree, w = RN16(rcp(sum)), P = RN16(e w). A NaN score takes the lower clamp + (min / max return the number).""" + s = np.asarray(s, np.float64) + if s.shape[-1] != 16: + raise ValueError("softmax_q7: 16 keys") + with np.errstate(over="ignore", invalid="ignore"): + u = np.fmin(np.fmax(f16(s * Q7_SCORE_SCALE), -Q7_SCORE_CLAMP), Q7_SCORE_CLAMP) + t = f16(u * -u + Q7_POLY_T) + p = f16(u * t + Q7_POLY_P) + pb = p.astype(np.float16).view(np.uint16) + e = (((pb & 0x03FF) << 5).astype(np.uint16)).view(np.float16).astype(np.float64) + + def pair(lo): # a_t = RN16(e_{lo+2t} + e_{lo+8+2t}), tree over t + a = [f16(e[..., lo + 2 * t] + e[..., lo + 8 + 2 * t]) for t in range(4)] + return f16(f16(a[0] + a[1]) + f16(a[2] + a[3])) + + total = f16(pair(0) + pair(1)) + w = f16(_rcp32(total)) + return f16(e * w[..., None]) + + +def gelu_q7(h): + """y = clamp(h, -2, 2); g = RN16(h RN16(1/2 + RN16(y RN16(a - RN16(b |y|))))), every step its own rounding.""" + h = np.asarray(h, np.float64) + with np.errstate(over="ignore", invalid="ignore"): + y = np.fmin(np.fmax(h, -2.0), 2.0) + return f16(h * f16(0.5 + f16(y * f16(Q7_GELU_A - f16(Q7_GELU_B * np.abs(y)))))) + + +def _score_op(B, u, model): + """Scores s[..., h, i, j] = one FP8 op of u_i and u_j (32 terms) started from B[h, i, j]; u [..., h, T, 32].""" + B = np.asarray(B, np.float64) + with np.errstate(over="ignore", invalid="ignore"): + if model == "exact": + return f16(B + np.einsum("...htd,...hsd->...hts", u, u)) + if model != "trunc13": + raise ValueError(f"unknown FP8 op model {model!r}") + prods = u[..., :, None, :] * u[..., None, :, :] # [..., h, T, T, 32] + acc = np.broadcast_to(B, prods.shape[:-1]) + for b0 in (0, 16): + acc = _trunc13_block(acc, prods[..., b0:b0 + 16]) + return acc + + +def _block_tokens_q7(xw, p, heads, k): + """The block in its FP16 form on gathered windows xw [..., T, C] (FP16 values; T = 16; C = 32, 64, 96 or 128; + heads of 32), FP8 ops by the model k["mm8"] (MM8_MODELS, default "exact"): + norm (norm_q7, gamma1) -> FP8 -> per head u = FP8(Wqk chain from 0), v = Wv chain from 0 (FP16), scores = one FP8 op + u u started from B, weights softmax_q7, o = RN16(P v) -> FP8; z = the Wp chain (one 32-wide chunk per head, head + order) started from RN16(bp + x); norm (gamma2) -> FP8; the MLP in 32-wide hidden slices, ascending: h = the W1 + slice's chain started from b1's slice, gelu_q7, FP8, added by W2's matching chunk onto the chain started from + RN16(b2 + z). Returns the FP16 block output. The matrices are rounded to FP8 here (no change for an FP8 file).""" + C = xw.shape[-1] + T = xw.shape[-2] + if C not in NORM_WIDTHS or T != 16 or k["head_dim"] != 32: + raise ValueError("_block_tokens_q7: C 32, 64, 96 or 128, 4 x 4 windows, heads of 32") + model = k.get("mm8", "exact") + lead = xw.shape[:-2] + x = np.asarray(xw, np.float64) + W = {n: q8(p[n]) for n in ("Wqk", "Wv", "Wp", "W1", "W2")} + with np.errstate(over="ignore", invalid="ignore"): + a8 = q8(norm_q7(x, gamma_q7(p["gamma1"]))) + u = np.stack([q8(mm8(0.0, a8, W["Wqk"][h], model)) for h in range(heads)], axis=-3) # [..., h, T, 32] + v = np.stack([mm8(0.0, a8, W["Wv"][h], model) for h in range(heads)], axis=-3) + s = _score_op(p["B"], u, model) + o = f16(np.einsum("...hts,...hsd->...htd", softmax_q7(s), v)) + o8 = q8(np.moveaxis(o, -3, -2).reshape(lead + (T, heads * 32))) + z = mm8(f16(p["bp"] + x), o8, W["Wp"], model) + m8 = q8(norm_q7(z, gamma_q7(p["gamma2"]))) + acc = f16(p["b2"] + z) + for sl in range(W["W1"].shape[0] // 32): + r = slice(32 * sl, 32 * sl + 32) + hs = mm8(p["b1"][r], m8, W["W1"][r], model) + acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, r], model) + return acc + + def window_grid(H, W, offset, win): ox, oy = offset nwx = (W + ox + win - 1) // win @@ -101,10 +327,12 @@ def f(q): raise ValueError(f"unknown border {border!r}") -def swin_block(x, p, offset, heads, consts, border="mirror", halo=0, max_tokens=1 << 16): +def swin_block(x, p, offset, heads, consts, border="mirror", halo=0, max_tokens=1 << 16, skip=None): """One block on a whole map x [H, W, C] (border "input": [H + 2 halo, W + 2 halo, C], the grid in the middle); - window members outside the grid per border_index. Returns the block output on the grid [H, W, C] in float64, not - yet rounded.""" + window members outside the grid per border_index. skip [H, W, C] (FP8 values): each member's input is + RN16(x member + skip member), the skip member taken with its own border, the mirror about the grid size (a + member outside the grid reads x by the block's border and skip by the mirror). Returns the block output on the + grid [H, W, C] in float64, not yet rounded (the FP16 form: FP16).""" x = np.asarray(x, dtype=np.float64) h = halo if border == "input" else 0 H, W, C = x.shape[0] - 2 * h, x.shape[1] - 2 * h, x.shape[2] @@ -113,12 +341,16 @@ def swin_block(x, p, offset, heads, consts, border="mirror", halo=0, max_tokens= rows, cols = window_grid(H, W, offset, win) out = np.full((H, W, C), np.nan) fr, fc = border_index(border, H, h), border_index(border, W, h) + if consts.get("mm8") == "trunc13": + max_tokens = min(max_tokens, 4096) rows_per_chunk = max(1, max_tokens // (win * win * cols.shape[0])) for l0 in range(0, rows.shape[0], rows_per_chunk): r = rows[l0:l0 + rows_per_chunk] rr = fr(r)[:, None, :, None] cc = fc(cols)[None, :, None, :] xw = x[rr, cc] + if skip is not None: + xw = f16(xw + skip[_swin.mirror(r, H)[:, None, :, None], _swin.mirror(cols, W)[None, :, None, :]]) nl, nk = xw.shape[:2] x2 = _block_tokens(xw.reshape(nl, nk, win * win, C), p, heads, consts).reshape(nl, nk, win, win, C) rv = (r >= 0) & (r < H) @@ -151,15 +383,16 @@ def depth_to_space(v): return out -def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): - """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp (None: - no clamp) and round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" +def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP, model="exact"): + """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (one FP8 op per group of at most + 32, by the FP8 op model); then clamp (None: no clamp) and round to FP8. With groups (20, 20, 20, 20) this is + clean/reference/encoder.py project.""" W = np.asarray(W, dtype=np.float64) z = np.asarray(z, dtype=np.float64) s = np.broadcast_to(np.asarray(b, dtype=np.float64), z.shape[:-1] + (W.shape[0],)) lo = 0 for g in groups: - s = f16(s + z[..., lo:lo + g] @ W[:, lo:lo + g].T) + s = _op8(s, z[..., lo:lo + g], W[:, lo:lo + g], model) lo += g if clamp is not None: s = np.clip(s, -clamp, clamp) @@ -172,19 +405,24 @@ def _out_round(y, mode): return _round(y, mode) -def linear(x, W, b, acc=None, out_round="q8"): - """out_round(acc(W x + b)) with exact sums; acc "f16" rounds the sum to FP16 first.""" - y = np.asarray(x, dtype=np.float64) @ np.asarray(W, dtype=np.float64).T + np.asarray(b, dtype=np.float64) +def linear(x, W, b, acc=None, out_round="q8", model="exact"): + """out_round(acc(W x + b)); acc "f16": the FP8 op chain started from b (by the FP8 op model); acc None: exact + sums.""" if acc == "f16": - y = f16(y) - elif acc not in (None, "none"): + y = mm8(np.asarray(b, dtype=np.float64), x, W, model) + elif acc in (None, "none"): + y = np.asarray(x, dtype=np.float64) @ np.asarray(W, dtype=np.float64).T + np.asarray(b, dtype=np.float64) + else: raise ValueError(f"unknown accumulation {acc!r}") return _out_round(y, out_round) def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): - """Interprets an op list over named maps; returns the last op's output.""" + """Interprets an op list over named maps; returns the last op's output. FP8 ops by the table's "mm8" model.""" last = None + model = t.get("mm8", "exact") + if model not in MM8_MODELS: + raise ValueError(f"unknown FP8 op model {model!r}") for op in ops: k = op["op"] st = params[op.get("params", default_stage)] @@ -205,12 +443,21 @@ def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): elif k == "select": out = maps[op["src"]][..., op["lo"]:op["hi"]] elif k == "linear": - out = linear(maps[op["src"]], st[op["w"]], st[op["b"]], op.get("acc"), op.get("out_round", "q8")) + out = linear(maps[op["src"]], st[op["w"]], st[op["b"]], op.get("acc"), op.get("out_round", "q8"), model) elif k == "swin": - consts = _table.block_consts(t, op["params"]) + consts = dict(_table.block_consts(t, op["params"]), mm8=model) off = _table.block_offset(t, op["offsets"], parity) border = op.get("border", "mirror") - y = swin_block(maps[op["src"]], st, off, op["heads"], consts, border, maps.get("_halo", 0)) + skip = None + if op.get("skip"): + skip = maps.get(op["skip"]) + if skip is None: + raise ValueError(f"{default_stage}: the skip map {op['skip']!r} is needed (the encoder's block " + "output)") + skip = np.asarray(skip, dtype=np.float64) + if skip.shape != maps[op["src"]].shape: + raise ValueError(f"swin skip: shapes {skip.shape} and {maps[op['src']].shape}") + y = swin_block(maps[op["src"]], st, off, op["heads"], consts, border, maps.get("_halo", 0), skip=skip) if border == "input": maps["_halo"] = 0 out = _round(y, op.get("out_round", "q8")) @@ -222,9 +469,10 @@ def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): parts.append(np.zeros(parts[0].shape[:-1] + (pad - c,))) out = np.concatenate(parts, axis=-1) elif k == "project": - out = project_groups(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("clamp")) + out = project_groups(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("clamp"), model) elif k == "dec_project": - out = dec_project(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("out_round", "q8")) + out = dec_project(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("out_round", "q8"), + model) elif k == "pick": out = np.asarray(maps[op["src"]])[..., list(op["values"])] else: @@ -274,7 +522,9 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): if maps.pop("_halo"): raise ValueError("encoder: the per-pixel halo was not consumed by a block") m_near, m_far, m_center = _enc.motion_surfaces(x) - res = {"features": feats.astype(np.float32), + skip = maps.get(t.get("skip_map")) + res = {"skip": None if skip is None else np.asarray(skip, np.float32), + "features": feats.astype(np.float32), "m_near": f16(m_near).astype(np.float32), "m_far": f16(m_far).astype(np.float32), "m_center": f16(m_center).astype(np.float32)} @@ -285,15 +535,15 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): # ---- decoder -------------------------------------------------------------------------------------------------------- -def dec_project(Z, W, b, groups=(32, 32), out_round="q8"): +def dec_project(Z, W, b, groups=(32, 32), out_round="q8", model="exact"): """Z [h, w, 64], W [4, 32, 64], b [4, 32] -> [h, w, 128] (channel 32 s + c): Model M's dec0 projection arithmetic - (clean/reference/decoder.py project) over all 4 x 32 rows.""" + (clean/reference/decoder.py project) over all 4 x 32 rows, one FP8 op per group (by the FP8 op model).""" Z = np.asarray(Z, dtype=np.float64) W = np.asarray(W, dtype=np.float64).reshape(128, -1) s = np.broadcast_to(np.asarray(b, dtype=np.float64).reshape(128), Z.shape[:-1] + (128,)) lo = 0 for g in groups: - s = f16(s + Z[..., lo:lo + g] @ W[:, lo:lo + g].T) + s = _op8(s, Z[..., lo:lo + g], W[:, lo:lo + g], model) lo += g return _round(s, out_round) @@ -348,7 +598,8 @@ def decode(x, params, parity, t=_table.TABLE, keep=False): if len(t["dec_ops"][-1]["values"]) != _table.DEC_PIXEL_VALUES[kind]: raise ValueError(f"dec_pixel {kind!r} needs {_table.DEC_PIXEL_VALUES[kind]} picked values") rw, rh = _dec.check_inputs(x)[:2] - maps = {"features": np.asarray(x["features"], dtype=np.float64)} + maps = {"features": np.asarray(x["features"], dtype=np.float64), + "skip": None if x.get("skip") is None else np.asarray(x["skip"], dtype=np.float64)} a = _run_ops(t["dec_ops"], maps, params, parity, t, "dec0") dec = params["dec0"] if kind == "l": @@ -396,6 +647,7 @@ def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): res[s] = o xd = dict(x_dec) xd["features"] = x + xd["skip"] = res["enc0"]["skip"] res["dec0"] = decode(xd, params, parity, t, keep) return res diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py index 13e02ecc..abbfc7b4 100644 --- a/clean/l/reference/table.py +++ b/clean/l/reference/table.py @@ -72,8 +72,8 @@ {"op": "dec_project", "src": "features", "dst": "yt", "w": "W", "b": "b", "groups": (32, 32), "out_round": "q8"}, {"op": "depth_to_space", "src": "yt", "dst": "y"}, - {"op": "swin", "src": "y", "dst": "y2", "params": "dec0_block", **C32_BLOCK, "offsets": "second", - "border": "coarse", "out_round": "f16"}, + {"op": "swin", "src": "y", "skip": "skip", "dst": "y2", "params": "dec0_block", **C32_BLOCK, + "offsets": "second", "border": "coarse", "out_round": "f16"}, {"op": "pick", "src": "y2", "values": L_DEC_VALUES}, ), "dec_pixel": "l", @@ -95,13 +95,27 @@ # Window offsets (ox, oy) of an L block for frame parity 0 and 1, in pixels; the blocks run on the padded grid F # and keep every grid position's output. "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, - # Per-block constants; a block's own entry (key = its params stage) wins over "default". - "consts": {"default": dict(M_BLOCK_CONSTS)}, + # Per-block constants; a block's own entry (key = its params stage) wins over "default". "form": "q7" = the + # block's FP16 form (oracle._block_tokens_q7: FP16 accumulators per 32-wide FP8 op, the polynomial-and-bit-shift + # attention weights, the FP16 GELU polynomial, the FP16 norm tree with sqrt(C) in the gain, residuals as + # accumulator starts), both of L's own blocks; "m" = Model M's block arithmetic (oracle._block_tokens). + "consts": {"default": dict(M_BLOCK_CONSTS, form="q7")}, + # The arithmetic inside one FP8 matrix op (oracle.MM8_MODELS): "exact" (the exact sum rounded once to FP16, the + # reference) or "trunc13" (two blocks of 16 products, terms truncated to 13 bits below the largest). Every record + # names the model it used. + "mm8": "exact", + # The encoder map the decoder block's skip input reads (the encoder block's output, FP8, no clamp). A swin op's + # "skip": each window member's input is RN16(src member + skip member); the skip member is read with the mirror + # about the grid size (fine positions), whatever the op's border (the decoder: "coarse" for the projection). + "skip_map": "x3", } DEC_PIXEL_VALUES = {"l": len(L_DEC_VALUES), "m": len(M_DEC0_VALUES)} -PROVISIONAL = {} +PROVISIONAL = { + "S8": "the ten shared stages run Model M's oracle (clean/reference/swin.py, Model M's block arithmetic); their FP16 " + "form is pending a decision", +} # Model M's reconstruction in the same terms (clean/reference/post.py reconstruct): for reductions and tests. M_RECON = {"rule": "d_over_render_at_least", "ratio": 3.0, "variants": ("3/2", "3/1"), "window": 3, @@ -199,7 +213,7 @@ def encoder_plan(t=TABLE): def decoder_plan(t=TABLE): """(op, level, channels) after each decoder op, from the op list alone.""" - return _plan(t["dec_ops"], {"features": ("token", 64)}, "decoder", last="pick") + return _plan(t["dec_ops"], {"features": ("token", 64), "skip": ("pixel", 32)}, "decoder", last="pick") def _plan(ops, maps, what, last): @@ -237,6 +251,8 @@ def _plan(ops, maps, what, last): raise ValueError(f"swin: border {border!r} needs a pixel-level map") if border == "input" and any(p[0] not in ("quant", "linear", "select", "concat") for p in plan): raise ValueError("swin: border 'input' needs only per-pixel ops before the block") + if op.get("skip") and maps.get(op["skip"]) != src: + raise ValueError(f"swin: the skip map {op['skip']!r} must match the input {src}") out = src elif k == "concat": parts = [maps[s] for s in op["srcs"]] diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index d42d00c2..6878b1b9 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -37,6 +37,7 @@ from clean.l.reference import oracle as L # noqa: E402 from clean.l.reference import table as T # noqa: E402 from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 from clean.reference import swin as MS # noqa: E402 from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 from clean.runner import packing as P # noqa: E402 @@ -304,8 +305,7 @@ def main(argv=None): if a.compile_only: return 0 res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) - with open(a.json, "w") as f: - json.dump(res, f, indent=1) + LR.write_l_record(res, a.json) print({k: v for k, v in res.items() if k not in ("c32", "l_blocks")}) lb = res["l_blocks"] print({k: lb[k] for k in ("enc_total", "dec_total", "nc_cases", "nc_identical", "pass")}) diff --git a/clean/l/runner/l_d3d12_own_post_check.py b/clean/l/runner/l_d3d12_own_post_check.py index c491f0e4..6adf3b57 100644 --- a/clean/l/runner/l_d3d12_own_post_check.py +++ b/clean/l/runner/l_d3d12_own_post_check.py @@ -44,6 +44,7 @@ from clean.l.runner import l_frame_check as F # noqa: E402 from clean.l.runner import l_post_check as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 OPTIONS = "model=L;l_post=own" SENTINELS = {"d3d12_own": 0x5A, "d3d12_own_stages": 0x3C} @@ -136,8 +137,7 @@ def compare(build, json_path): all(r["pass"] for r in post_run.values()) and res["d3d12_default_vs_stages"]["identical_frames"] == n and caught and own["kernels"].get("l_post") == "l_post_recon") - with open(json_path, "w") as f: - json.dump(res, f, indent=1, default=lambda x: x.item() if hasattr(x, "item") else str(x)) + LR.write_l_record(res, json_path) print("L d3d12 l_post=own:", "pass" if res["pass"] else "FAIL", "frames", n) print(" against the HIP run (raw bytes): identical", res["vs_hip"]["identical_frames"], "of", n, "outputs compared", res["vs_hip"]["outputs_compared"], "differing", res["vs_hip"]["differing"]) diff --git a/clean/l/runner/l_default_post_check.py b/clean/l/runner/l_default_post_check.py index 4db982a7..b63d7cde 100644 --- a/clean/l/runner/l_default_post_check.py +++ b/clean/l/runner/l_default_post_check.py @@ -38,6 +38,7 @@ from clean.l.runner import l_frame_check as F # noqa: E402 from clean.l.runner import l_post_check as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 CONFIGS = {"own_stages": ("own", True, 0x3C)} # (A, "m_default", ran with l_post=m: no longer a runtime route) @@ -116,8 +117,7 @@ def compare(build, json_path): res["A_m_post"] = post_frames(md, params) keys.insert(0, "A_m_post") res["B_own_post"] = post_frames(own, params) - with open(json_path, "w") as f: - json.dump(res, f, indent=1, default=lambda x: x.item() if hasattr(x, "item") else str(x)) + LR.write_l_record(res, json_path) print("model source:", json.dumps(res["model_source"])) for key in keys: r = res[key] diff --git a/clean/l/runner/l_enc_check.py b/clean/l/runner/l_enc_check.py index 476f7ae6..31f997d0 100644 --- a/clean/l/runner/l_enc_check.py +++ b/clean/l/runner/l_enc_check.py @@ -30,6 +30,7 @@ from clean.l.reference import table as T # noqa: E402 from clean.l.runner import l_c32_check as C32 # noqa: E402 from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 from clean.reference import encoder as ME # noqa: E402 from clean.reference.fp8 import fp8_decode # noqa: E402 from clean.runner import enc0 as E0 # noqa: E402 @@ -121,8 +122,7 @@ def main(argv=None): ap.add_argument("--json", required=True) a = ap.parse_args(argv) res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) - with open(a.json, "w") as f: - json.dump(res, f, indent=1) + LR.write_l_record(res, a.json) print({"total": res["total"], "pass": res["pass"]}) for m in res["frames"]: if not m["pass"]: diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index 0b8d4902..b0215c84 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -356,7 +356,7 @@ def run_d3d12(build, special=True, stages=False): "ldr": LDR and model == "L", "notes": p.notes(), "model_source": p.model_source, "dw": p.dw, "dh": p.dh, "ow": p.ow, "oh": p.oh, - "taps": p.output_taps(), "frames": frames} + "taps": p.output_taps(), "frames": frames, "identity": capture_identity(build)} with open(rec_path(build, model, config), "wb") as f: pickle.dump(rec, f) print(model, config, "frames", len(frames), "post", rec["kernels"].get("post"), "pack", @@ -513,8 +513,8 @@ def model_pass(m): return (r["d3d12_per_stage"]["passed"] == n_frames[m] and r["m_post_reference"]["passed"] == n_frames[m] and r["d3d12_default_vs_stages"]["identical_frames"] == n_frames[m] and caught) res["pass"] = all(model_pass(m) for m in ("L", "M")) - with open(json_path, "w") as f: - json.dump(res, f, indent=1, default=lambda o: o.item() if hasattr(o, "item") else str(o)) + from clean.l.runner import l_record as LR + LR.write_l_record(res, json_path) for model in ("L", "M"): print(model, "D3D12 against HIP (raw bytes): frames identical", res[model]["identical_frames"], "of", len(res[model]["frames"]), "outputs compared", len(res[model]["outputs_compared"]), "differing", @@ -561,7 +561,7 @@ def run(build, rocm, special=True): "ldr": LDR and model == "L", "notes": p.notes(), "model_source": p.model_source, "dw": p.dw, "dh": p.dh, "ow": p.ow, "oh": p.oh, - "taps": p.output_taps(), "frames": frames} + "taps": p.output_taps(), "frames": frames, "identity": capture_identity(build)} with open(rec_path(build, model, config), "wb") as f: pickle.dump(rec, f) print(model, config, "frames", len(frames), "bneck", rec["kernels"].get("swin_bneck"), "post", @@ -569,6 +569,69 @@ def run(build, rocm, special=True): del p +# ---- identity of a recorded run ------------------------------------------------------------------------------------ + +def _sha256_file(path): + import hashlib + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def identity_files(build): + """The files a recorded run's identity covers: {name: path} (the runtime library, the oracle's parameters and + the model's source description, both written by --prepare).""" + return {"runtime_library": os.path.join(build, "wronggpu_runtime.dll"), + "params_pkl": os.path.join(model_dir(build), "params.pkl"), + "source_json": os.path.join(model_dir(build), "source.json")} + + +def capture_identity(build): + """At capture time (--run): the git commit of this checkout (git rev-parse HEAD, run here, not given by the + caller), whether its tracked files differ from that commit, and the SHA-256 of every identity file.""" + import subprocess + def git(*a): + try: + r = subprocess.run(["git", "-C", ROOT] + list(a), capture_output=True, text=True, + creationflags=0x08000000 if os.name == "nt" else 0) + except OSError as e: + raise RuntimeError(f"capture identity: git {a[0]} could not run ({e})") + if r.returncode != 0: + raise RuntimeError(f"capture identity: git {a[0]} failed ({r.returncode}): {r.stderr.strip()}") + return r.stdout.strip() + commit = git("rev-parse", "HEAD") + if not commit: + raise RuntimeError("capture identity: git rev-parse HEAD gave no commit") + ident = {"git_commit": commit, + "git_tracked_changes": bool(git("status", "--porcelain", "--untracked-files=no"))} + for name, path in identity_files(build).items(): + ident[name + "_sha256"] = _sha256_file(path) if os.path.exists(path) else None + return ident + + +def verify_identity(rec, build): + """Re-hash the identity files of `build` and compare them with the identity the recording captured. Returns + the verified identity; raises ValueError when the recording has none or anything differs.""" + ident = rec.get("identity") + if not ident or not ident.get("git_commit"): + raise ValueError("the recording carries no capture-time identity (record it again with --run)") + if not isinstance(ident.get("git_tracked_changes"), bool): + raise ValueError("the recording's identity has no tracked-changes flag (git_tracked_changes)") + for name, path in identity_files(build).items(): + now = _sha256_file(path) if os.path.exists(path) else None + if now is None or now != ident.get(name + "_sha256"): + raise ValueError(f"{name}: SHA-256 {now} differs from the recording's {ident.get(name + '_sha256')}") + return ident + + +def check_l_record(res_l): + """A Model L frame comparison record names the FP8 matrix-op model its oracle used, in every frame too.""" + from clean.l.runner import l_record as LR + return LR.check_l_record(res_l, frames_required=True) + + # ---- --compare (CPU) ------------------------------------------------------------------------------------------------ def _f16_steps(a, b): @@ -642,8 +705,9 @@ def post_fp16(got, ref, model="M"): return m -def compare_model(rec, params, post_ref=None, bneck_unit=False): - """post_ref "M": the post against Model M's reference whatever the model (a record of a run with Model M's post). +def compare_model(rec, params, post_ref=None, bneck_unit=False, table=None): + """table: Model L's structure table for the oracle (default: clean/l/reference/table.py TABLE with the kernels' + GELU for Model M's block form). post_ref "M": the post against Model M's reference whatever the model (a record of a run with Model M's post). bneck_unit: b0 ... b5 as one unit (a bottleneck megakernel writes only b5): the reference chained from the runtime's enc2.merged, b5 under the per-stage rule.""" from clean.l.reference import oracle as L @@ -655,8 +719,9 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False): from clean.reference import post as Po from clean.reference import swin as S model = rec["model"] - table = copy.deepcopy(T.TABLE) - table["consts"]["default"]["gelu"] = "sigmoid" # the kernels' GELU (clean/l/runner/l_c32_check.py CONSTS) + if table is None: + table = copy.deepcopy(T.TABLE) + table["consts"]["default"]["gelu"] = "sigmoid" # the kernels' GELU (clean/l/runner/l_c32_check.py CONSTS) D = (rec["dw"], rec["dh"]) res, prev, prev_jitter, n, f = [], None, None, 0, 0 prev_size = None @@ -669,6 +734,8 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False): jp = tuple(float(v) for v in (fr["jitter"] if eff_reset else prev_jitter)) info = out["state"]["info"] m = {"frame": k, "render": [rw, rh], "reset": eff_reset, "parity": f} + if model == "L": + m["fp8_op_model"] = table.get("mm8", "exact") # the effective model, also when defaulted offsets_ok = all(tuple(info["offsets"][s]) == tuple(S.window_offset(s, f)) for s in T.SHARED_STAGES) m["state"] = {"n_parity_reset": (info["n"], info["parity"], bool(info["reset"])) == (n, f, eff_reset), "jitter_prev": bool(np.array_equal(np.float32(info["jitter_prev"]), np.float32(jp))), @@ -700,6 +767,7 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False): x0["motion_history"] = np.zeros_like(x0["motion_history"]) x0["reduced_history"] = np.zeros_like(reduced) ref = L.encode(x0, params, f, t=table) if model == "L" else ME.encode(x0, params["enc0"]) + enc_skip = ref.get("skip") if model == "L" else None # the encoder block output the decoder block reads m["enc0"] = {"features": fp8_stage(out["enc0"]["features"], ref["features"])} for s in ("m_near", "m_far", "m_center"): m["enc0"][s] = motion_surface(out["enc0"][s], ref[s]) @@ -725,7 +793,7 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False): xd = {"features": out["dec1"]["out"], "motion": out["enc0"]["m_near"], "latent_prev": out["state"]["latent_prev"] if not eff_reset else np.zeros((rh, rw, 4), np.float32), "jitter": fr["jitter"], "jitter_prev": np.array(jp), "reset": eff_reset, "feature_size": (fw, fh), - "recon_size": D} + "recon_size": D, "skip": enc_skip} ref = L.decode(xd, params, f, t=table) if model == "L" else MD.decode(xd, params["dec0"]) dec = out["dec0"] m["dec0"] = {"codes_A": codes(dec["A"], ref["A"], 255.0), "codes_B": codes(dec["B"], ref["B"], 255.0), @@ -810,6 +878,7 @@ def summary(frames): def compare(build, json_path): + from clean.l.reference import table as T with open(os.path.join(model_dir(build), "params.pkl"), "rb") as f: params = pickle.load(f) res = {} @@ -836,6 +905,9 @@ def unwritten(k, n): "pass": (all(m["pass"] for m in frames) and all(f["identical"] for f in same) and all(v["caught"] for v in neg.values()))} if model == "L": # Model L's run-level limit on post values more than one ulp off (provisional) + res[model]["fp8_op_model"] = T.TABLE.get("mm8", "exact") + res[model]["identity"] = recs["stages"].get("identity") + check_l_record(res[model]) res[model]["post_run"] = {k: LP.l_post_run([m["post"][k] for m in frames if k in m["post"]]) for k in ("history", "display")} res[model]["pass"] = res[model]["pass"] and all(r["pass"] for r in res[model]["post_run"].values()) @@ -845,8 +917,8 @@ def unwritten(k, n): with open(src) as f: res["L"]["source"] = json.load(f) print("Model L source:", json.dumps(res["L"]["source"])) - with open(json_path, "w") as f: - json.dump(res, f, indent=1, default=lambda o: o.item() if hasattr(o, "item") else str(o)) + from clean.l.runner import l_record as LR + LR.write_l_record(res, json_path) for model in ("L", "M"): print(model, "pass" if res[model]["pass"] else "FAIL", json.dumps(res[model]["summary"])) print(" default kernels against the six-launch bottleneck (raw bytes):", diff --git a/clean/l/runner/l_ops_check.py b/clean/l/runner/l_ops_check.py index 2552d854..065b1d56 100644 --- a/clean/l/runner/l_ops_check.py +++ b/clean/l/runner/l_ops_check.py @@ -36,6 +36,7 @@ from clean.l.reference import oracle as L # noqa: E402 from clean.l.runner import l_c32_check as C32 # noqa: E402 from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 CASES = [(16, 24), (32, 64), (8, 8), (64, 96)] # pixel maps (rows, cols), even sides @@ -312,8 +313,7 @@ def main(argv=None): ap.add_argument("--json", required=True) a = ap.parse_args(argv) res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) - with open(a.json, "w") as f: - json.dump(res, f, indent=1) + LR.write_l_record(res, a.json) print({k: v for k, v in res.items() if k.endswith("_total") or k == "pass"}) for k in ("relu", "f16", "project", "dec_project", "coef", "boundary", "coef_pack12"): for m in res[k]: diff --git a/clean/l/runner/l_q7_negative_control.py b/clean/l/runner/l_q7_negative_control.py new file mode 100644 index 00000000..3787647c --- /dev/null +++ b/clean/l/runner/l_q7_negative_control.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): negative control of the oracle's FP16 block form. One recorded run of l_frame_check (HIP, every +kernel separate: lframe_L_stages.pkl) is checked twice in one process, frame by frame on the run's own stage inputs +(l_frame_check.compare_model): +- "model_m_form": the oracle as it was before the FP16 form (both of Model L's blocks in Model M's block arithmetic + with the kernels' GELU, the decoder block without the skip input); +- "fp16_form": the oracle as it is (clean/l/reference/table.py TABLE: both blocks in the FP16 form, the decoder block + reading the encoder block's output as its skip, which this check takes from the encoder oracle on the run's own + inputs), with the FP8 matrix-op model --mm8. +Kernels that compute Model M's block form pass the first and must fail the second; kernels in the FP16 form the other +way round. Identity: the recording carries what --run captured (l_frame_check.capture_identity: the checkout's git +commit from git rev-parse HEAD, and the SHA-256 of the runtime library, params.pkl and source.json); this check re-hashes +the files it loads and refuses when the recording has no identity or any hash differs (l_frame_check.verify_identity). +Output: a JSON record with counts and digests only (the verified identity, the model's and map's SHA-256 from +source.json, the SHA-256 of the recorded run file and of the frames' inputs and outputs), the command line (the build +folder as ), the FP8 op model of each oracle, and the per-frame alignment checks (frame counter, parity, reset, +window offsets, as compare_model checks them). + + python clean/l/runner/l_frame_check.py --build --prepare --model # CPU + python clean/l/runner/l_frame_check.py --build --run # GPU + python clean/l/runner/l_q7_negative_control.py --build --json out.json # CPU +""" +import argparse +import copy +import hashlib +import json +import os +import pickle +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_frame_check as F # noqa: E402 + +DEC0_RULE = "A, B and latent codes and C values each at most 0.5 % differing (l_frame_check dec0)" + + +def model_m_form_table(): + """The oracle before the FP16 form: Model M's block arithmetic (the kernels' GELU) in both blocks, no skip.""" + t = copy.deepcopy(T.TABLE) + t["consts"]["default"]["form"] = "m" + t["consts"]["default"]["gelu"] = "sigmoid" + for op in t["dec_ops"]: + op.pop("skip", None) + return t + + +def fp16_form_table(mm8): + t = copy.deepcopy(T.TABLE) + t["mm8"] = mm8 + return t + + +def _sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _sha256_arrays(arrays): + h = hashlib.sha256() + for a in arrays: + a = np.ascontiguousarray(a) + h.update(str(a.dtype).encode() + str(a.shape).encode()) + h.update(a.tobytes()) + return h.hexdigest() + + +def frame_digests(rec): + """SHA-256 over every frame's inputs (sorted keys) and over every frame's recorded outputs (raw bytes).""" + ins, outs = [], [] + for fd in rec["frames"]: + for k in sorted(fd["inputs"]): + v = fd["inputs"][k] + ins.append(np.asarray(v) if not isinstance(v, (bool, int, float)) else np.asarray([v])) + for k in sorted(fd["raw"]): + outs.append(np.frombuffer(bytes(fd["raw"][k]), np.uint8)) + return {"inputs": _sha256_arrays(ins), "outputs_raw": _sha256_arrays(outs)} + + +def dec0_rows(frames): + rows = [] + for m in frames: + d = m["dec0"] + rows.append({"frame": m["frame"], "render": m["render"], "parity": m["parity"], "reset": m["reset"], + "alignment": m["state"], "enc0_features_pass": m["enc0"]["features"]["pass"], + "dec0_pass": d["pass"], + "dec0_differ": {k: v["differ"] for k, v in d.items() if isinstance(v, dict)}, + "dec0_values": {k: v["values"] for k, v in d.items() if isinstance(v, dict)}}) + return rows + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--config", default="stages", help="the recorded run (lframe_L_.pkl)") + ap.add_argument("--mm8", default="exact", choices=L.MM8_MODELS) + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + run_path = F.rec_path(a.build, "L", a.config) + with open(run_path, "rb") as f: + rec = pickle.load(f) + with open(os.path.join(F.model_dir(a.build), "params.pkl"), "rb") as f: + params = pickle.load(f)["L"] + with open(os.path.join(F.model_dir(a.build), "source.json")) as f: + source = json.load(f) + try: + ident = F.verify_identity(rec, a.build) # the files this check loads are the ones the run used + except ValueError as e: + print("REFUSED:", e) + return 2 + cmd = [os.path.basename(sys.argv[0])] + [("" if x == a.build else ("" if x == a.json else x)) + for x in (argv if argv is not None else sys.argv[1:])] + res = {"what": "negative control of Model L's oracle block form: one recorded run checked against the Model " + "M-form oracle and the FP16-form oracle in one process", + "command": " ".join(["python", "clean/l/runner/" + cmd[0]] + cmd[1:]), + "runtime_commit": ident["git_commit"], + "identity_verified": ident, + "recorded_run": {"file": os.path.basename(run_path), "sha256": _sha256_file(run_path), + "frames": len(rec["frames"]), "model": rec["model"], "kernels": rec["kernels"], + "frame_sha256": frame_digests(rec)}, + "model_source": source, "fp8_op_model": a.mm8, "dec0_rule": DEC0_RULE, "forms": {}} + for name, table in (("model_m_form", model_m_form_table()), ("fp16_form", fp16_form_table(a.mm8))): + frames = F.compare_model(rec, params, table=table) + rows = dec0_rows(frames) + res["forms"][name] = {"fp8_op_model": table["mm8"], + "dec0_frames_passed": sum(r["dec0_pass"] for r in rows), + "enc0_frames_passed": sum(r["enc0_features_pass"] for r in rows), + "alignment_frames_ok": sum(bool(r["alignment"]["pass"]) for r in rows), + "frames": rows} + print(name, "dec0 frames passed", res["forms"][name]["dec0_frames_passed"], "of", len(rows), + "| enc0", res["forms"][name]["enc0_frames_passed"], "| alignment ok", + res["forms"][name]["alignment_frames_ok"], flush=True) + from clean.l.runner import l_record as LR + LR.write_l_record(res, a.json, table=fp16_form_table(a.mm8)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_record.py b/clean/l/runner/l_record.py new file mode 100644 index 00000000..b024d4e0 --- /dev/null +++ b/clean/l/runner/l_record.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: MIT +"""Model L comparison records (issue #89): every record a Model L check writes against the oracle names the FP8 +matrix-op model the oracle used (clean/l/reference/table.py "mm8", oracle.MM8_MODELS), also when it is the default. +Every such writer goes through write_l_record, which adds the effective model and refuses a record that names none or +names one its frames contradict. + + write_l_record(res, path) # the default table's model + write_l_record(res, path, table=t) # the model of the table the oracle ran with +""" +import json + + +def effective_mm8(table=None): + """The FP8 op model the oracle uses with `table` (default: the shipped table), also when the table leaves it out.""" + from clean.l.reference import table as T + return (T.TABLE if table is None else table).get("mm8", "exact") + + +def check_l_record(res, frames_required=False): + """Raises ValueError unless `res` names a known FP8 op model; every frame that names one (all frames when + frames_required) must name the same.""" + from clean.l.reference import oracle as L + model = res.get("fp8_op_model") + if model not in L.MM8_MODELS: + raise ValueError("Model L comparison record without its FP8 op model (fp8_op_model)") + for m in res.get("frames", []): + if (frames_required or "fp8_op_model" in m) and m.get("fp8_op_model") != model: + raise ValueError(f"frame {m.get('frame')}: FP8 op model missing or different") + return res + + +def write_l_record(res, path, table=None): + """Adds the effective FP8 op model to the record (top level), checks it and writes the JSON file.""" + res["fp8_op_model"] = effective_mm8(table) + check_l_record(res) + with open(path, "w", newline="\n") as f: + json.dump(res, f, indent=1, default=lambda o: o.item() if hasattr(o, "item") else str(o)) + f.write("\n") + return res diff --git a/clean/l/tests/test_l_pack.py b/clean/l/tests/test_l_pack.py index 2baf6b98..eb8796b6 100644 --- a/clean/l/tests/test_l_pack.py +++ b/clean/l/tests/test_l_pack.py @@ -174,13 +174,15 @@ def test_unpacked_weights_reproduce_the_oracle(self): h1 = L.linear(qpad, W1, b1, "f16", "relu_q8") self.assertTrue(np.array_equal(h1, L.linear(m["q"], enc["Win1"], enc["bin1"], "f16", "relu_q8"))) h2 = L.linear(h1, W2, b2, "f16", "none") - h3 = q8(L.swin_block(h2, params["enc0_block"], T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS, + h3 = q8(L.swin_block(h2, params["enc0_block"], T.block_offset(T.TABLE, "first", 0), 2, + T.block_consts(T.TABLE, "enc0_block"), "input", T.block_halo(T.TABLE, "first"))) self.assertTrue(np.array_equal(h3, m["x3"])) Wp, bpj = LP.unpack_project(blobs["enc0.project"], 128, 64) feats = L.project_groups(L.space_to_depth(h3), Wp, bpj, (32, 32, 32, 32), 6.28125) self.assertEqual(feats.astype(np.float32).tobytes(), out["features"].tobytes()) xd = MD.random_inputs(rng, rw=36, rh=20) + xd["skip"] = out["skip"] # the encoder block output (same frame size) dout = L.decode(xd, params, 0, keep=True) Wd, bd = LP.unpack_dec_project(blobs["dec0.project"]) y = L.dec_project(xd["features"], Wd, bd) diff --git a/clean/l/tests/test_l_q7_known.py b/clean/l/tests/test_l_q7_known.py new file mode 100644 index 00000000..ff448e19 --- /dev/null +++ b/clean/l/tests/test_l_q7_known.py @@ -0,0 +1,393 @@ +# SPDX-License-Identifier: MIT +"""Known answers for the FP16 block form of Model L's oracle (clean/l/reference/oracle.py, form "q7"), with nonzero +weights. The expected values are worked out here independently of the oracle: by hand (the block scenario) or with an +exact rational reference (fractions.Fraction with its own round-to-nearest-even to FP16 and FP8). Every check has +negative controls: the oracle mutated in one way (chunk order, a missing chunk, a rounding moved across a chunk +boundary, the wrong W2 slice, per-product accumulation, ties away from zero, subnormals flushed, overflow saturated, +the attention-weight and GELU constants) must fail it. + + python -m unittest clean.l.tests.test_l_q7_known +""" +import contextlib +import inspect +import math +import textwrap +import unittest +from fractions import Fraction as Fr + +import numpy as np + +from clean.l.reference import oracle as L +from clean.l.reference import table as T +from clean.reference.fp8 import f16 as F16 + +# ---- independent exact reference -------------------------------------------------------------------------------------- + + +def _floor_log2(a): + """floor(log2 a) of a positive Fraction.""" + e = a.numerator.bit_length() - a.denominator.bit_length() + while Fr(2) ** e > a: + e -= 1 + while Fr(2) ** (e + 1) <= a: + e += 1 + return e + + +def rn16(v): + """Round a Fraction to FP16, nearest even, subnormals kept, overflow to infinity (as a float).""" + v = Fr(v) + if v == 0: + return 0.0 + a = abs(v) + if a >= 65520: # halfway between 65504 and 2^16 and above: infinity + return math.copysign(math.inf, v) + q = Fr(2) ** (max(_floor_log2(a), -14) - 10) + return float(math.copysign(round(a / q) * q, v)) # Fraction round(): half to even + + +def rq8(v): + """Round to FP8 E4M3FN, nearest even, saturating at +-448 (as a float).""" + v = Fr(v) + if v == 0: + return 0.0 + a = abs(v) + q = Fr(2) ** (max(_floor_log2(a), -6) - 3) + return float(math.copysign(min(round(a / q) * q, Fr(448)), v)) + + +def bits16(x): + return int(np.array([x], np.float16).view(np.uint16)[0]) + + +def from_bits16(b): + return float(np.array([b], np.uint16).view(np.float16)[0]) + + +def ref_weights(s): + """Attention weights of one row of 16 FP16 scores, written out step by step.""" + c, lim = Fr(from_bits16(0x2466)), Fr(from_bits16(0x3873)) + t0, p0 = Fr(from_bits16(0x3B6B)), Fr(from_bits16(0x3D80)) + e = [] + for si in s: + u = Fr(rn16(Fr(si) * c)) + u = min(max(u, -lim), lim) + t = Fr(rn16(u * -u + t0)) # one rounding (fused) + p = rn16(u * t + p0) + e.append(Fr(from_bits16((bits16(p) & 0x03FF) << 5))) + a = [Fr(rn16(e[2 * k] + e[8 + 2 * k])) for k in range(4)] + b = [Fr(rn16(e[2 * k + 1] + e[9 + 2 * k])) for k in range(4)] + A = Fr(rn16(Fr(rn16(a[0] + a[1])) + Fr(rn16(a[2] + a[3])))) + B = Fr(rn16(Fr(rn16(b[0] + b[1])) + Fr(rn16(b[2] + b[3])))) + total = rn16(A + B) + w = Fr(rn16(Fr(float(np.float32(1) / np.float32(total))))) # (the reciprocal's last bit is free) + return [rn16(ei * w) for ei in e] + + +def ref_gelu(h): + h = Fr(h) + y = min(max(h, Fr(-2)), Fr(2)) + a, b = Fr(from_bits16(0x3698)), Fr(from_bits16(0x2D30)) + inner = Fr(rn16(a - Fr(rn16(b * abs(y))))) + return rn16(h * Fr(rn16(Fr(1, 2) + Fr(rn16(y * inner))))) + + +def ref_norm(x, gamma): + """RMS norm of one pixel (FP16 values x, C channels) as the tree: lane t, half (even / odd channel), tiles.""" + C = len(x) + q = [Fr(rn16(Fr(v) * Fr(v))) for v in x] + tiles = C // 8 + + def grp(ps): + if len(ps) == 1: + return ps[0] + if len(ps) == 2: + return Fr(rn16(ps[0] + ps[1])) + if len(ps) == 3: + return Fr(rn16(Fr(rn16(ps[0] + ps[1])) + ps[2])) + return Fr(rn16(Fr(rn16(ps[0] + ps[1])) + Fr(rn16(ps[2] + ps[3])))) + + def lane(t, odd): + Q = [q[8 * j + 2 * t + odd] for j in range(tiles)] + left = grp([Fr(rn16(Q[j] + Q[j + 2])) for j in range(0, tiles, 4)]) + right = grp([Fr(rn16(Q[j] + Q[j + 2])) for j in range(1, tiles, 4)]) + return Fr(rn16(left + right)) + + def half(odd): + e = [lane(t, odd) for t in range(4)] + return Fr(rn16(Fr(rn16(e[0] + e[1])) + Fr(rn16(e[2] + e[3])))) + + s = rn16(Fr(rn16(half(0) + half(1))) + Fr(1, 2 ** 13)) + r = Fr(rn16(Fr(float(np.float32(1) / np.sqrt(np.float32(s)))))) + g = [Fr(rn16(Fr(float(np.float32(gm) * np.sqrt(np.float32(C)))))) for gm in gamma] + return [rn16(Fr(v) * Fr(rn16(r * gi))) for v, gi in zip(x, g)] + + +# ---- mutation of the oracle ------------------------------------------------------------------------------------------- + +@contextlib.contextmanager +def mutated(name, old, new): + """The oracle's function `name` with `old` replaced by `new` in its source (exactly one occurrence).""" + f = getattr(L, name) + src = textwrap.dedent(inspect.getsource(f)) + if src.count(old) != 1: + raise AssertionError(f"mutation of {name}: {old!r} occurs {src.count(old)} times") + ns = dict(L.__dict__) + exec(compile(src.replace(old, new), f"", "exec"), ns) + setattr(L, name, ns[name]) + try: + yield + finally: + setattr(L, name, f) + + +@contextlib.contextmanager +def patched(name, value): + old = getattr(L, name) + setattr(L, name, value) + try: + yield + finally: + setattr(L, name, old) + + +def f16_ties_away(x): + x = np.asarray(x, np.float64) + r = F16(x) + lo, hi = np.nextafter(r.astype(np.float16), np.float16(-np.inf)), np.nextafter(r.astype(np.float16), np.float16(np.inf)) + lo, hi = lo.astype(np.float64), hi.astype(np.float64) + tie_lo = np.isfinite(r) & (x - lo == r - x) & (np.abs(lo) > np.abs(r)) + tie_hi = np.isfinite(r) & (hi - x == x - r) & (np.abs(hi) > np.abs(r)) + return np.where(tie_lo, lo, np.where(tie_hi, hi, r)) + + +def f16_flush(x): + r = F16(x) + return np.where(np.abs(r) < 2.0 ** -14, 0.0 * r, r) + + +def f16_saturate(x): + r = F16(x) + return np.where(np.isinf(r) & ~np.isinf(np.asarray(x, np.float64)), np.copysign(65504.0, r), r) + + +# ---- the checks (each returns True when the oracle gives the known answer) --------------------------------------------- + +def chain_case(): + """K = 64 (two chunks). Start 1024 (FP16 spacing 1): chunk 0 adds 0.75 -> RN16(1024.75) = 1025, chunk 1 adds 0.5 -> + RN16(1025.5) = 1026 (tie, even). Reversed: 1024.5 -> 1024 (tie, even), 1024.75 -> 1025. Exact across chunks: + 1025.25 -> 1025. Chunk 1 missing: 1025.""" + a = np.zeros(64) + a[3], a[40] = 0.75, 0.5 + W = np.zeros((2, 64)) + W[:, 3], W[:, 40] = 1.0, 1.0 + W[1] *= -1.0 + got = L.mm8(np.array([1024.0, -1024.0]), a, W) + return list(got) == [1026.0, -1026.0] + + +def op_case(): + """One 32-term op, start 2048 (FP16 spacing 2), four products of 0.5: the exact sum 2050 is FP16. A rounding after + each product (2048.5 -> 2048, tie even) gives 2048. Sixteen products of 0.125 (2050 exactly) too; truncating the + terms to 13 bits below 2^11 (multiples of 0.25) would drop them (2048).""" + a = np.zeros(32) + a[:4] = 0.5 + W = np.ones((1, 32)) + b = np.zeros(32) + b[:16] = 0.125 + return (L.mm8(np.array([2048.0]), a, W)[0] == 2050.0 and + L.mm8(np.array([2048.0]), b, W)[0] == 2050.0) + + +def trunc13_case(): + """The truncating model on the same op: 0.125 is below the quantum 2^(11 - 13) = 0.25, so 2048 stays; 0.25 terms + survive (sixteen give 2052); block 2's start is block 1's FP16 result.""" + b = np.zeros(32) + b[:16] = 0.125 + c = np.zeros(32) + c[:16] = 0.25 + W = np.ones((1, 32)) + return (L.mm8(np.array([2048.0]), b, W, "trunc13")[0] == 2048.0 and + L.mm8(np.array([2048.0]), c, W, "trunc13")[0] == 2052.0) + + +PROBES = ( # (k, a, b) products, start, result bits (exact model), result bits (trunc13 model) + (((0, 2.0 ** -2, 2.0 ** -9), (1, 2.0 ** -9, 2.0 ** -9)), 1.0, 0x3C01, 0x3C00), # 1 + 2^-11 + 2^-18 + (((0, -2.0 ** -3, 2.0 ** -9), (1, -2.0 ** -9, 2.0 ** -9)), 1.0, 0x3BFF, 0x3C00), # 1 - 2^-12 - 2^-18 + (((0, 1.0, 1.0), (1, 2.0 ** -2, 2.0 ** -9), (16, 2.0 ** -2, 2.0 ** -9)), 0.0, 0x3C01, 0x3C00), # across the blocks + # 1 + 2^-11 (a tie) + 0.75 x 2^-13: truncated to 0 the tie goes to even (1.0); rounded it would pass the tie + (((0, 2.0 ** -2, 2.0 ** -9), (1, 1.5 * 2.0 ** -6, 2.0 ** -8)), 1.0, 0x3C01, 0x3C00), +) + + +def probe_case(model): + """Single ops that tell the two models apart (worked out by hand: the exact sum is just past / short of an FP16 tie; + truncation to 13 bits below 2^0 drops the 2^-18 term, and block 1 rounds 1 + 2^-11 to 1 before block 2).""" + ok = True + for prods, start, b_exact, b_trunc in PROBES: + a, W = np.zeros(32), np.zeros((1, 32)) + for k, x, w in prods: + a[k], W[0, k] = x, w + got = L.mm8(np.array([start]), a, W, model)[0] + ok = ok and bits16(got) == (b_exact if model == "exact" else b_trunc) + return ok + + +def rounding_edges_case(): + """FP16 ties to even (1024.5 -> 1024, 1025.5 -> 1026), subnormals kept (2.5 x 2^-24 -> 2 x 2^-24, 1.5 x 2^-24 -> 2^-23) + and overflow to infinity (65504 + 32): through one FP8 op and through the GELU (h -> h / 2 for tiny h).""" + W = np.ones((1, 32)) + a = np.zeros(32) + a[0] = 0.5 + t1 = L.mm8(np.array([1024.0]), a, W)[0] == 1024.0 and L.mm8(np.array([1025.0]), a, W)[0] == 1026.0 + a2 = np.zeros(32) + a2[0], a2[1] = 8.0, 4.0 # 32 + t2 = np.isposinf(L.mm8(np.array([65504.0]), a2, np.array([[4.0, 0.0] + [0.0] * 30]))[0]) + t2 = t2 and np.isposinf(L.mm8(np.array([65504.0]), np.r_[8.0, 4.0, np.zeros(30)], np.r_[2.0, 4.0, np.zeros(30)][None])[0]) + g = L.gelu_q7(np.array([5 * 2.0 ** -24, 3 * 2.0 ** -24])) + t3 = list(g) == [rn16(Fr(5, 2 ** 25)), rn16(Fr(3, 2 ** 25))] == [2 * 2.0 ** -24, 2.0 ** -23] + return bool(t1 and t2 and t3) + + +def block_case(): + """A whole block (C 32, 2 heads, MLP 128) with nonzero weights, worked out by hand: + x = 1 everywhere, gamma1 = gamma2 = 1: the norm gives 1 (sum 32, r = RN16(rsqrt 32) = 0.1767578125, + RN16(r x 5.65625) = 1.0); Wqk = 0 and B = 0: every score 0, every weight 1/16; Wv[h][0][0] = 1: v = 1 in channel 0 of + each head, o = v; Wp[:, 0] = 0.75 (head 0), Wp[:, 32] = 0.5 (head 1), bp = 1023: z = 1024 -> 1025 -> 1026 (head order; + reversed 1025, exact 1025); norm 2 of 1026: the squares overflow, r = 0, the MLP input is 0; b1 slices 2, 1, 2, -2: + GELU 2, 0.8310546875 -> FP8 0.8125, 2, 0; W2[:, 0] = 0.375 (slice 0), W2[:, 64] = 0.25 (slice 2), b2 = -2: the chain + from RN16(-2 + 1026) = 1024 adds 0.75 -> 1025, 0, 0.5 -> 1026 (tie, even), 0. Output 1026 in every channel.""" + p = {k: np.zeros_like(v, dtype=np.float64) for k, v in L.random_block_params(np.random.default_rng(0), 32, 2, 128).items()} + p["gamma1"] = p["gamma2"] = np.ones(32) + p["Wv"][0, 0, 0] = p["Wv"][1, 0, 0] = 1.0 + p["Wp"][:, 0], p["Wp"][:, 32] = 0.75, 0.5 + p["bp"] = np.full(32, 1023.0) + p["b1"] = np.repeat([2.0, 1.0, 2.0, -2.0], 32) + p["W2"][:, 0], p["W2"][:, 64] = 0.375, 0.25 + p["b2"] = np.full(32, -2.0) + x = np.ones((2, 16, 32)) + got = L._block_tokens(x, p, 2, T.block_consts(T.TABLE, "dec0_block")) + return bool(np.all(got == 1026.0)) + + +def weights_case(): + rng = np.random.default_rng(11) + rows = [np.zeros(16), np.r_[60.0, -60.0, np.zeros(14)], L.f16(rng.normal(0, 8, 16)), L.f16(rng.normal(0, 20, 16)), + np.linspace(-31, 31, 16)] + return all(list(L.softmax_q7(r)) == ref_weights(r) for r in rows) + + +def gelu_case(): + h = np.r_[L.f16(np.linspace(-2.5, 2.5, 41)), 1.0, 0.3, -0.7, 1.9] + h = L.f16(h) + return list(L.gelu_q7(h)) == [ref_gelu(v) for v in h] and L.gelu_q7(np.array([1.0]))[0] == 0.8310546875 + + +def norm_case(): + rng = np.random.default_rng(12) + ok = True + for C in (32, 64, 96, 128): + for scale in (0.5, 4.0, 16.0): # (sums of squares below the FP16 range) + x = L.f16(rng.normal(0, scale, C)) + g = L.f16(rng.uniform(0.5, 1.5, C)) + ok = ok and list(L.norm_q7(x, L.gamma_q7(g))) == ref_norm(x, g) + return ok + + +# ---- tests ------------------------------------------------------------------------------------------------------------ + +REV_CHUNKS = ("for c0 in range(0, W8.shape[1], 32):", "for c0 in reversed(range(0, W8.shape[1], 32)):") +MISS_CHUNK = ("for c0 in range(0, W8.shape[1], 32):", "for c0 in range(0, max(32, W8.shape[1] - 32), 32):") +EXACT_ACROSS = (" acc = start\n for c0 in range(0, W8.shape[1], 32):\n" + " acc = _op8(acc, a8[..., c0:c0 + 32], W8[:, c0:c0 + 32], model)\n return acc", + " return f16(np.asarray(start, np.float64) + a8 @ W8.T)") +PER_PRODUCT = (" return f16(start + a8 @ W8.T)", + " acc = start\n for i in range(a8.shape[-1]):\n" + " acc = f16(acc + a8[..., i:i + 1] @ W8[:, i:i + 1].T)\n return acc") +W2_SLICE = ('acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, r], model)', + 'acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, slice(32 * ((sl + 1) % 4), 32 * ((sl + 1) % 4) + 32)], model)') +MLP_REVERSED = ('for sl in range(W["W1"].shape[0] // 32):', 'for sl in reversed(range(W["W1"].shape[0] // 32)):') + + +class Q7KnownAnswerTest(unittest.TestCase): + def test_reference_rounding(self): + """The rational reference itself: ties to even, subnormals, overflow, FP8 saturation.""" + self.assertEqual((rn16(Fr(2049, 2)), rn16(Fr(2051, 2))), (1024.0, 1026.0)) + self.assertEqual((rn16(Fr(5, 2 ** 25)), rn16(Fr(3, 2 ** 25))), (2 * 2.0 ** -24, 2.0 ** -23)) + self.assertEqual((rn16(65519), rn16(65520)), (65504.0, math.inf)) + self.assertEqual((rq8(Fr(17, 16)), rq8(Fr(19, 16)), rq8(500)), (1.0, 1.25, 448.0)) + self.assertEqual(L.f16(np.array([1024.5, 65520.0]))[0], 1024.0) + + def test_chunk_chain(self): + self.assertTrue(chain_case()) + for name, mut in (("mm8", REV_CHUNKS), ("mm8", MISS_CHUNK), ("mm8", EXACT_ACROSS)): + with self.subTest(mutant=mut[1][:40]), mutated(name, *mut): + self.assertFalse(chain_case()) + + def test_one_op_accumulation(self): + self.assertTrue(op_case()) + with mutated("_op8", *PER_PRODUCT): + self.assertFalse(op_case()) + with mutated("mm8", 'model="exact"', 'model="trunc13"'): # the other model as the default + self.assertFalse(op_case()) + + def test_trunc13_model(self): + self.assertTrue(trunc13_case()) + with mutated("_trunc13_block", "emax - 13", "emax - 16"): + self.assertFalse(trunc13_case()) + + def test_model_probes(self): + self.assertTrue(probe_case("exact") and probe_case("trunc13")) + with mutated("_trunc13_block", "np.trunc(terms / quantum)", "np.round(terms / quantum)"): + self.assertFalse(probe_case("trunc13")) # rounding instead of truncation + with mutated("_op8", " for b0 in (0, 16):\n acc = _trunc13_block(acc, prods[..., b0:b0 + 16])", + " acc = _trunc13_block(acc, prods)"): # one block of 32 instead of two of 16 + self.assertFalse(probe_case("trunc13")) + with mutated("_trunc13_block", "emax - 13", "emax - 20"): + self.assertFalse(probe_case("trunc13")) + + def test_rounding_edges(self): + self.assertTrue(rounding_edges_case()) + for f in (f16_ties_away, f16_flush, f16_saturate): + with self.subTest(mutant=f.__name__), patched("f16", f): + self.assertFalse(rounding_edges_case()) + + def test_block_by_hand(self): + self.assertTrue(block_case()) + for name, mut in (("mm8", REV_CHUNKS), ("mm8", MISS_CHUNK), ("mm8", EXACT_ACROSS), + ("_block_tokens_q7", W2_SLICE), ("_block_tokens_q7", MLP_REVERSED)): + with self.subTest(mutant=mut[1][:40]), mutated(name, *mut): + self.assertFalse(block_case()) + + def test_attention_weights(self): + self.assertTrue(weights_case()) + for name, value in (("Q7_POLY_T", L._h(0x3B6C)), ("Q7_POLY_P", L._h(0x3D81)), ("Q7_SCORE_SCALE", L._h(0x2467)), + ("Q7_SCORE_CLAMP", L._h(0x3800))): + with self.subTest(mutant=name), patched(name, value): + self.assertFalse(weights_case()) + with mutated("softmax_q7", "(pb & 0x03FF) << 5", "(pb & 0x03FF) << 4"): + self.assertFalse(weights_case()) + with mutated("softmax_q7", "e[..., lo + 8 + 2 * t]", "e[..., lo + 1 + 2 * t]"): + self.assertFalse(weights_case()) + + def test_gelu(self): + self.assertTrue(gelu_case()) + for name, value in (("Q7_GELU_A", L._h(0x3699)), ("Q7_GELU_B", L._h(0x2D31))): + with self.subTest(mutant=name), patched(name, value): + self.assertFalse(gelu_case()) + with mutated("gelu_q7", "f16(h * f16(0.5", "f16(y * f16(0.5"): # the clamped y outside instead of h + self.assertFalse(gelu_case()) + + def test_norm(self): + self.assertTrue(norm_case()) + with mutated("norm_q7", "range(1, tiles, 4)", "range(0, tiles, 4)"): + self.assertFalse(norm_case()) + with mutated("gamma_q7", "np.sqrt(np.float32(C))", "np.sqrt(np.float32(32.0))"): + self.assertFalse(norm_case()) + with mutated("norm_q7", "return f16(f16(e[0] + e[1]) + f16(e[2] + e[3]))", + "return f16(f16(f16(e[0] + e[1]) + e[2]) + e[3])"): + self.assertFalse(norm_case()) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_l_recon.py b/clean/l/tests/test_l_recon.py index 39c94735..81461bfc 100644 --- a/clean/l/tests/test_l_recon.py +++ b/clean/l/tests/test_l_recon.py @@ -302,7 +302,7 @@ def test_oracle_entry_uses_the_post_scale(self): x = _inputs(rng) params = {"post": {"s": np.float32(2.9796233)}} self.assertTrue(_same(L.reconstruct(x, params)["display"], R.reconstruct(x, 2.9796233)["display"])) - self.assertEqual(T.PROVISIONAL, {}) + self.assertEqual(set(T.PROVISIONAL), {"S8"}) # only the shared stages' block form is pending if __name__ == "__main__": diff --git a/clean/l/tests/test_l_records.py b/clean/l/tests/test_l_records.py new file mode 100644 index 00000000..5f281cb3 --- /dev/null +++ b/clean/l/tests/test_l_records.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: MIT +"""Model L comparison records (clean/l/runner/l_frame_check.py): every record names the FP8 op model its oracle used, +and a recorded run's capture-time identity is verified against the files a check loads. + + python -m unittest clean.l.tests.test_l_records +""" +import copy +import json +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +from clean.l.reference import table as T +from clean.l.runner import l_frame_check as F +from clean.l.runner import l_record as LR + + +def _read(path): + with open(path, encoding="utf-8") as f: + return f.read() + + +def _load(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _write(path, data): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + + +class RecordTest(unittest.TestCase): + def test_record_without_the_op_model_fails(self): + rec = {"frames": [{"frame": 0}, {"frame": 1}]} + with self.assertRaises(ValueError): + F.check_l_record(rec) # no fp8_op_model at all + rec["fp8_op_model"] = "exact" + with self.assertRaises(ValueError): + F.check_l_record(rec) # frames without it + for m in rec["frames"]: + m["fp8_op_model"] = "exact" + F.check_l_record(rec) + rec["frames"][1]["fp8_op_model"] = "trunc13" + with self.assertRaises(ValueError): + F.check_l_record(rec) # a frame with another model + with self.assertRaises(ValueError): + F.check_l_record({"fp8_op_model": "other", "frames": []}) + + def test_default_op_model_is_named(self): + self.assertEqual(T.TABLE.get("mm8"), "exact") + + def test_identity_is_verified(self): + with tempfile.TemporaryDirectory() as d: + files = F.identity_files(d) + for name, path in files.items(): + _write(path, name.encode()) + ident = F.capture_identity(d) + self.assertTrue(ident["git_commit"]) # from git rev-parse HEAD in this checkout + self.assertEqual(len(ident["runtime_library_sha256"]), 64) + rec = {"identity": ident} + self.assertEqual(F.verify_identity(rec, d), ident) + _write(files["params_pkl"], b"other parameters") + with self.assertRaises(ValueError): + F.verify_identity(rec, d) # params.pkl replaced after the capture + _write(files["params_pkl"], b"params_pkl") + os.remove(files["runtime_library"]) + with self.assertRaises(ValueError): + F.verify_identity(rec, d) # the library gone + with self.assertRaises(ValueError): + F.verify_identity({}, d) # a recording without identity + + +class IdentityFailureTest(unittest.TestCase): + def _files(self, d): + for name, path in F.identity_files(d).items(): + _write(path, name.encode()) + + def test_git_status_failure_fails_the_capture(self): + real = subprocess.run + + def fake(cmd, *a, **k): + if "status" in cmd: + return subprocess.CompletedProcess(cmd, 128, "", "fatal: not a git repository") + return real(cmd, *a, **k) + + with tempfile.TemporaryDirectory() as d: + self._files(d) + with mock.patch("subprocess.run", side_effect=fake): + with self.assertRaises(RuntimeError): + F.capture_identity(d) + + def test_git_failure_to_run_fails_the_capture(self): + with tempfile.TemporaryDirectory() as d: + self._files(d) + with mock.patch("subprocess.run", side_effect=OSError("git not found")): + with self.assertRaises(RuntimeError): + F.capture_identity(d) + + def test_tracked_changes_flag_required(self): + with tempfile.TemporaryDirectory() as d: + self._files(d) + ident = F.capture_identity(d) + self.assertIsInstance(ident["git_tracked_changes"], bool) + for bad in ({k: v for k, v in ident.items() if k != "git_tracked_changes"}, + dict(ident, git_tracked_changes=None), dict(ident, git_tracked_changes="no")): + with self.subTest(flag=bad.get("git_tracked_changes", "missing")): + with self.assertRaises(ValueError): + F.verify_identity({"identity": bad}, d) + + +# The checks that write a Model L comparison record against the oracle (each goes through LR.write_l_record). +L_ORACLE_EMITTERS = ("l_frame_check", "l_enc_check", "l_ops_check", "l_c32_check", "l_default_post_check", + "l_d3d12_own_post_check", "l_q7_negative_control") + + +class WriterTest(unittest.TestCase): + def test_defaulted_table_names_exact(self): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "r.json") + LR.write_l_record({"frames": [{"frame": 0}]}, p) # the shipped table + self.assertEqual(_load(p)["fp8_op_model"], "exact") + t = copy.deepcopy(T.TABLE) + del t["mm8"] # a table that leaves it out + LR.write_l_record({"x": 1}, p, table=t) + self.assertEqual(_load(p)["fp8_op_model"], "exact") + t["mm8"] = "trunc13" + LR.write_l_record({"x": 1}, p, table=t) + self.assertEqual(_load(p)["fp8_op_model"], "trunc13") + with self.assertRaises(ValueError): # a frame contradicting it + LR.write_l_record({"frames": [{"frame": 0, "fp8_op_model": "trunc13"}]}, p) + + def test_every_l_oracle_emitter_writes_through_the_writer(self): + here = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "runner") + for name in L_ORACLE_EMITTERS: + with self.subTest(emitter=name): + src = _read(os.path.join(here, name + ".py")) + self.assertIn("write_l_record(", src) + self.assertNotIn("json.dump(res", src) # no record written around the writer + # and no other runner compares with the oracle without being listed + for fn in os.listdir(here): + if fn.endswith(".py") and fn[:-3] not in L_ORACLE_EMITTERS: + src = _read(os.path.join(here, fn)) + uses_oracle = any(s in src for s in ("L.encode(", "L.decode(", "compare_model(", "L.swin_block(", + "L.linear(", "L.project_groups(", "L.dec_project(")) + with self.subTest(runner=fn): + self.assertFalse(uses_oracle and "json.dump(" in src) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py index 0f368d0f..da08dc26 100644 --- a/clean/l/tests/test_l_reference.py +++ b/clean/l/tests/test_l_reference.py @@ -38,16 +38,16 @@ def _same(a, b): return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes() -def _block_by_windows(value_at, H, W, off, p, heads=2): +def _block_by_windows(value_at, H, W, off, p, heads=2, consts=T.M_BLOCK_CONSTS): """A block written window by window: value_at(y, x) gives any window member's input (inside or outside the - grid); outputs are kept on the H x W grid. Float64, not rounded.""" + grid); outputs are kept on the H x W grid. Float64, not rounded (form "m"; the "q7" form returns FP16).""" p = {k: np.asarray(v, np.float64) for k, v in p.items()} ox, oy = off out = None for y0 in range(-oy, H, 4): for x0 in range(-ox, W, 4): pos = [(y, x) for y in range(y0, y0 + 4) for x in range(x0, x0 + 4)] - y = L._block_tokens(np.stack([value_at(a, b) for a, b in pos]), p, heads, T.M_BLOCK_CONSTS) + y = L._block_tokens(np.stack([value_at(a, b) for a, b in pos]), p, heads, consts) if out is None: out = np.full((H, W, y.shape[-1]), np.nan) for i, (a, b) in enumerate(pos): @@ -195,7 +195,8 @@ def chain(yy, xx): Tn[abs(a) % 16, abs(b) % 16]) for b in range(-2, fw + 2)]) for a in range(-2, fh + 2)]) self.assertTrue(_same(m["q"], q)) - h = q8(_block_by_windows(chain, fh, fw, T.block_offset(T.TABLE, "first", parity), params["enc0_block"])) + h = q8(_block_by_windows(chain, fh, fw, T.block_offset(T.TABLE, "first", parity), params["enc0_block"], + consts=T.block_consts(T.TABLE, "enc0_block"))) np.testing.assert_array_equal(m["x3"], h) self.assertFalse(is_fp8_exact(chain(3, 4))) z = np.zeros((fh // 2, fw // 2, 128)) @@ -264,8 +265,9 @@ def test_default_decoder(self): rng = np.random.default_rng(10) x = MD.random_inputs(rng, rw=30, rh=18) params = L.random_params(rng) - out = L.decode(x, params, 1, keep=True) fw, fh = x["feature_size"] + x["skip"] = q8(rng.normal(0, 2, (fh, fw, 32))) # the encoder block's output (FP8) + out = L.decode(x, params, 1, keep=True) m = out["maps"] self.assertEqual(m["yt"].shape, (fh // 2, fw // 2, 128)) self.assertEqual(m["y2"].shape, (fh, fw, 32)) @@ -273,18 +275,26 @@ def test_default_decoder(self): y = L.dec_project(x["features"], params["dec0"]["W"], params["dec0"]["b"]) hw = fw // 2 - def member(yy, xx): # coarse mirror of the token, sub-pixel kept (rows stay inside) + def proj(yy, xx): # coarse mirror of the token, sub-pixel kept (rows stay inside) q = xx // 2 if not 0 <= q < hw: q = min(abs(q), fw - abs(q) - 2) return y[yy // 2, q, 32 * ((xx % 2) + 2 * (yy % 2)):][:32] - self.assertTrue(_same(member(0, -1), member(0, 3)) and _same(member(1, fw), member(1, fw - 4))) - p = L.f16(_block_by_windows(member, fh, fw, T.block_offset(T.TABLE, "second", 1), params["dec0_block"])) + def member(yy, xx): # + the skip at the fine mirror of the position + fx = abs(xx) if abs(xx) < fw else 2 * fw - 2 - abs(xx) + return L.f16(proj(yy, xx) + x["skip"][yy, fx]) + + self.assertTrue(_same(proj(0, -1), proj(0, 3)) and _same(proj(1, fw), proj(1, fw - 4))) + self.assertFalse(_same(member(0, -1), L.f16(proj(0, 3) + x["skip"][0, 3]))) # the skip: fine 1, not 3 + k = T.block_consts(T.TABLE, "dec0_block") # the FP16 form ("q7") + self.assertEqual(k["form"], "q7") + p = L.f16(_block_by_windows(member, fh, fw, T.block_offset(T.TABLE, "second", 1), params["dec0_block"], + consts=k)) np.testing.assert_array_equal(m["y2"], p) # FP16 block output, no FP8 self.assertFalse(is_fp8_exact(p)) fine = L.f16(L.swin_block(L.depth_to_space(y), params["dec0_block"], T.block_offset(T.TABLE, "second", 1), - 2, T.M_BLOCK_CONSTS)) + 2, k, skip=x["skip"])) self.assertFalse(_same(fine, p)) # not a fine-pixel mirror self.assertTrue(_same(fine[:, 2:-2], p[:, 2:-2])) # ... only the edge windows differ a = p[:18, :30][..., [0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5]].astype(np.float32) @@ -409,5 +419,91 @@ def test_table_copy(self): T.table(nonsense=1) +class Q7FormTest(unittest.TestCase): + """The width-32 block's FP16 form (oracle._block_tokens_q7) and its pieces: known answers.""" + + def test_attention_weights_known_answers(self): + P = L.softmax_q7(np.zeros(16)) # s = 0: p = 1.375, e = 0.125, sum 2, w = 0.5 + np.testing.assert_array_equal(P, np.full(16, 0.0625)) + big = L.softmax_q7(np.array([1e4] + [0.0] * 15)) + clamp = L.softmax_q7(np.array([60.0] + [0.0] * 15)) # 60 c is past the clamp already + np.testing.assert_array_equal(big, clamp) + nan = L.softmax_q7(np.array([np.nan] + [0.0] * 15)) # min / max return the number: the lower clamp + low = L.softmax_q7(np.array([-1e4] + [0.0] * 15)) + np.testing.assert_array_equal(nan, low) + self.assertTrue(np.all(L.softmax_q7(np.linspace(-80, 80, 16)) > 0)) + + def test_attention_weight_bits(self): + """bits(e) = (mantissa bits of p) << 5 for p in [1, 2): e grows with the score.""" + s = np.linspace(-30, 30, 16) + u = np.fmin(np.fmax(L.f16(s * L.Q7_SCORE_SCALE), -L.Q7_SCORE_CLAMP), L.Q7_SCORE_CLAMP) + p = L.f16(u * L.f16(u * -u + L.Q7_POLY_T) + L.Q7_POLY_P) + self.assertTrue(np.all((p >= 1) & (p < 2))) + P = L.softmax_q7(s) + self.assertTrue(np.all(np.diff(P) > 0)) + + def test_gelu_known_answers(self): + h = np.array([2.0, 3.5, 100.0, -2.0, -3.0, 0.0, 1.0]) + g = L.gelu_q7(h) + np.testing.assert_array_equal(g[:3], h[:3]) # g = h from 2 on + np.testing.assert_array_equal(g[3:6], 0.0) # 0 from -2 down, and at 0 + self.assertEqual(g[6], 0.8310546875) # 1 x (1/2 + 1 x (a - b)) + self.assertTrue(np.isnan(L.gelu_q7(np.array([np.nan]))[0])) + + def test_norm_known_answers(self): + self.assertEqual(float(L.gamma_q7(np.array([1.0]), 32)[0]), 5.65625) # RN16(float32(1) x sqrtf(32)) + x = np.ones(32) + r = L.f16(np.float32(1) / np.sqrt(np.float32(32))) # the sum is 32, + 2^-13 rounds back to 32 + np.testing.assert_array_equal(L.norm_q7(x, L.gamma_q7(np.ones(32))), L.f16(r * 5.65625)) + z = np.zeros(32) + np.testing.assert_array_equal(L.norm_q7(z, L.gamma_q7(np.ones(32))), z) # 0 x r, r from the epsilon + + def test_norm_sum_order(self): + """The even / odd lane tree: a value that the plain sequential FP16 sum loses survives the tree.""" + x = np.zeros(32) + x[0], x[1] = 256.0, 256.0 # squares 65536 -> FP16 inf: both halves overflow + self.assertTrue(np.all(L.norm_q7(x, L.gamma_q7(np.ones(32)))[2:] == 0)) + x = np.zeros(32) + x[0] = 64.0 # 4096 in the even half + x[1:32:2] = 0.5 # 16 x 0.25 = 4 in the odd half (each lane 1.0 exactly) + q_even, q_odd = 4096.0, 4.0 + s = L.f16(L.f16(q_even + q_odd) + L.Q7_NORM_EPS) + r = L.f16(np.float32(1) / np.sqrt(np.float32(s))) + np.testing.assert_array_equal(L.norm_q7(x, L.gamma_q7(np.ones(32))), L.f16(x * L.f16(r * 5.65625))) + + def test_zero_weights_pass_the_input(self): + """All weights and biases 0: z = RN16(0 + x), the MLP adds nothing, the block returns its FP16 input.""" + rng = np.random.default_rng(3) + p = {k: np.zeros_like(v) for k, v in L.random_block_params(rng, 32, 2, 128).items()} + p["gamma1"] = p["gamma2"] = np.ones(32) + x = L.f16(rng.normal(0, 3, (5, 16, 32))) + k = T.block_consts(T.TABLE, "dec0_block") + np.testing.assert_array_equal(L._block_tokens(x, p, 2, k), x) + + def test_residual_starts_and_chunk_order(self): + """The output is the W2 chain started from RN16(b2 + z), z the Wp chain started from RN16(bp + x): with only + bp and b2 set, the block adds them with those two roundings.""" + rng = np.random.default_rng(4) + p = {k: np.zeros_like(v) for k, v in L.random_block_params(rng, 32, 2, 128).items()} + p["gamma1"] = p["gamma2"] = np.ones(32) + p["bp"] = L.f16(rng.normal(0, 1, 32)) + p["b2"] = L.f16(rng.normal(0, 1, 32)) + x = L.f16(rng.normal(0, 30, (3, 16, 32))) + got = L._block_tokens(x, p, 2, T.block_consts(T.TABLE, "dec0_block")) + np.testing.assert_array_equal(got, L.f16(p["b2"] + L.f16(p["bp"] + x))) + + def test_both_blocks_use_the_fp16_form(self): + self.assertEqual(T.block_consts(T.TABLE, "dec0_block")["form"], "q7") + self.assertEqual(T.block_consts(T.TABLE, "enc0_block")["form"], "q7") + self.assertEqual(T.TABLE["mm8"], "exact") + rng = np.random.default_rng(5) + p = L.random_block_params(rng, 32, 2, 128) + x = q8(rng.normal(0, 2, (4, 16, 32))) + m = L._block_tokens(x, p, 2, T.M_BLOCK_CONSTS) + q = L._block_tokens(x, p, 2, T.block_consts(T.TABLE, "dec0_block")) + self.assertFalse(np.array_equal(L.f16(m), q)) + self.assertTrue(np.array_equal(L.f16(q), q)) # the FP16 form's output is FP16 + + if __name__ == "__main__": unittest.main()